src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     def test_GET_etags(self):
 904
 905         def _check_etags(uri):
 906             d1 = _get_etag(uri)
 907             d2 = _get_etag(uri, 'json')
 908             d = defer.DeferredList([d1, d2], consumeErrors=True)
 909             def _check(results):
 910                 # All deferred must succeed
 911                 self.failUnless(all([r[0] for r in results]))
 912                 # the etag for the t=json form should be just like the etag
 913                 # fo the default t='' form, but with a 'json' suffix
 914                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 915             d.addCallback(_check)
 916             return d
 917
 918         def _get_etag(uri, t=''):
 919             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 920             d = self.GET(targetbase, return_response=True, followRedirect=True)
 921             def _just_the_etag(result):
 922                 data, response, headers = result
 923                 etag = headers['etag'][0]
 924                 if uri.startswith('URI:DIR'):
 925                     self.failUnless(etag.startswith('DIR:'), etag)
 926                 return etag
 927             return d.addCallback(_just_the_etag)
 928
 929         # Check that etags work with immutable directories
 930         (newkids, caps) = self._create_immutable_children()
 931         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 932                       simplejson.dumps(newkids))
 933         d.addCallback(_check_etags)
 934
 935         # Check that etags work with immutable files
 936         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 937
 938         # use the ETag on GET
 939         def _check_match(ign):
 940             uri = "/uri/%s" % self._bar_txt_uri
 941             d = self.GET(uri, return_response=True)
 942             # extract the ETag
 943             d.addCallback(lambda (data, code, headers):
 944                           headers['etag'][0])
 945             # do a GET that's supposed to match the ETag
 946             d.addCallback(lambda etag:
 947                           self.GET(uri, return_response=True,
 948                                    headers={"If-None-Match": etag}))
 949             # make sure it short-circuited (304 instead of 200)
 950             d.addCallback(lambda (data, code, headers):
 951                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 952             return d
 953         d.addCallback(_check_match)
 954         return d
 955
 956     # TODO: version of this with a Unicode filename
 957     def test_GET_FILEURL_save(self):
 958         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 959                      return_response=True)
 960         def _got((res, statuscode, headers)):
 961             content_disposition = headers["content-disposition"][0]
 962             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 963             self.failUnlessIsBarDotTxt(res)
 964         d.addCallback(_got)
 965         return d
 966
 967     def test_GET_FILEURL_missing(self):
 968         d = self.GET(self.public_url + "/foo/missing")
 969         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 970         return d
 971
 972     def test_GET_FILEURL_info_mdmf(self):
 973         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 974         def _got(res):
 975             self.failUnlessIn("mutable file (mdmf)", res)
 976             self.failUnlessIn(self._quux_txt_uri, res)
 977             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 978         d.addCallback(_got)
 979         return d
 980
 981     def test_GET_FILEURL_info_mdmf_readonly(self):
 982         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 983         def _got(res):
 984             self.failUnlessIn("mutable file (mdmf)", res)
 985             self.failIfIn(self._quux_txt_uri, res)
 986             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 987         d.addCallback(_got)
 988         return d
 989
 990     def test_GET_FILEURL_info_sdmf(self):
 991         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 992         def _got(res):
 993             self.failUnlessIn("mutable file (sdmf)", res)
 994             self.failUnlessIn(self._baz_txt_uri, res)
 995         d.addCallback(_got)
 996         return d
 997
 998     def test_GET_FILEURL_info_mdmf_extensions(self):
 999         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1000         def _got(res):
1001             self.failUnlessIn("mutable file (mdmf)", res)
1002             self.failUnlessIn(self._quux_txt_uri, res)
1003             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1004         d.addCallback(_got)
1005         return d
1006
1007     def test_PUT_overwrite_only_files(self):
1008         # create a directory, put a file in that directory.
1009         contents, n, filecap = self.makefile(8)
1010         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1011         d.addCallback(lambda res:
1012             self.PUT(self.public_url + "/foo/dir/file1.txt",
1013                      self.NEWFILE_CONTENTS))
1014         # try to overwrite the file with replace=only-files
1015         # (this should work)
1016         d.addCallback(lambda res:
1017             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1018                      filecap))
1019         d.addCallback(lambda res:
1020             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1021                  "There was already a child by that name, and you asked me "
1022                  "to not replace it",
1023                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1024                  filecap))
1025         return d
1026
1027     def test_PUT_NEWFILEURL(self):
1028         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1029         # TODO: we lose the response code, so we can't check this
1030         #self.failUnlessReallyEqual(responsecode, 201)
1031         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1032         d.addCallback(lambda res:
1033                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1034                                                       self.NEWFILE_CONTENTS))
1035         return d
1036
1037     def test_PUT_NEWFILEURL_not_mutable(self):
1038         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1039                      self.NEWFILE_CONTENTS)
1040         # TODO: we lose the response code, so we can't check this
1041         #self.failUnlessReallyEqual(responsecode, 201)
1042         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1043         d.addCallback(lambda res:
1044                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1045                                                       self.NEWFILE_CONTENTS))
1046         return d
1047
1048     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1049         # this should get us a few segments of an MDMF mutable file,
1050         # which we can then test for.
1051         contents = self.NEWFILE_CONTENTS * 300000
1052         d = self.PUT("/uri?format=mdmf",
1053                      contents)
1054         def _got_filecap(filecap):
1055             self.failUnless(filecap.startswith("URI:MDMF"))
1056             return filecap
1057         d.addCallback(_got_filecap)
1058         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1059         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1060         return d
1061
1062     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1063         contents = self.NEWFILE_CONTENTS * 300000
1064         d = self.PUT("/uri?format=sdmf",
1065                      contents)
1066         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1067         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1068         return d
1069
1070     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1071         contents = self.NEWFILE_CONTENTS * 300000
1072         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1073                                     400, "Bad Request", "Unknown format: foo",
1074                                     self.PUT, "/uri?format=foo",
1075                                     contents)
1076
1077     def test_PUT_NEWFILEURL_range_bad(self):
1078         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1079         target = self.public_url + "/foo/new.txt"
1080         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1081                              "501 Not Implemented",
1082                              "Content-Range in PUT not yet supported",
1083                              # (and certainly not for immutable files)
1084                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1085                              headers=headers)
1086         d.addCallback(lambda res:
1087                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1088         return d
1089
1090     def test_PUT_NEWFILEURL_mutable(self):
1091         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1092                      self.NEWFILE_CONTENTS)
1093         # TODO: we lose the response code, so we can't check this
1094         #self.failUnlessReallyEqual(responsecode, 201)
1095         def _check_uri(res):
1096             u = uri.from_string_mutable_filenode(res)
1097             self.failUnless(u.is_mutable())
1098             self.failIf(u.is_readonly())
1099             return res
1100         d.addCallback(_check_uri)
1101         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1102         d.addCallback(lambda res:
1103                       self.failUnlessMutableChildContentsAre(self._foo_node,
1104                                                              u"new.txt",
1105                                                              self.NEWFILE_CONTENTS))
1106         return d
1107
1108     def test_PUT_NEWFILEURL_mutable_toobig(self):
1109         # It is okay to upload large mutable files, so we should be able
1110         # to do that.
1111         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1112                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1113         return d
1114
1115     def test_PUT_NEWFILEURL_replace(self):
1116         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1117         # TODO: we lose the response code, so we can't check this
1118         #self.failUnlessReallyEqual(responsecode, 200)
1119         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1120         d.addCallback(lambda res:
1121                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1122                                                       self.NEWFILE_CONTENTS))
1123         return d
1124
1125     def test_PUT_NEWFILEURL_bad_t(self):
1126         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1127                              "PUT to a file: bad t=bogus",
1128                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1129                              "contents")
1130         return d
1131
1132     def test_PUT_NEWFILEURL_no_replace(self):
1133         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1134                      self.NEWFILE_CONTENTS)
1135         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1136                   "409 Conflict",
1137                   "There was already a child by that name, and you asked me "
1138                   "to not replace it")
1139         return d
1140
1141     def test_PUT_NEWFILEURL_mkdirs(self):
1142         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1143         fn = self._foo_node
1144         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1145         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1146         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1147         d.addCallback(lambda res:
1148                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1149                                                       self.NEWFILE_CONTENTS))
1150         return d
1151
1152     def test_PUT_NEWFILEURL_blocked(self):
1153         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1154                      self.NEWFILE_CONTENTS)
1155         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1156                   "409 Conflict",
1157                   "Unable to create directory 'blockingfile': a file was in the way")
1158         return d
1159
1160     def test_PUT_NEWFILEURL_emptyname(self):
1161         # an empty pathname component (i.e. a double-slash) is disallowed
1162         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1163                              "400 Bad Request",
1164                              "The webapi does not allow empty pathname components",
1165                              self.PUT, self.public_url + "/foo//new.txt", "")
1166         return d
1167
1168     def test_DELETE_FILEURL(self):
1169         d = self.DELETE(self.public_url + "/foo/bar.txt")
1170         d.addCallback(lambda res:
1171                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1172         return d
1173
1174     def test_DELETE_FILEURL_missing(self):
1175         d = self.DELETE(self.public_url + "/foo/missing")
1176         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1177         return d
1178
1179     def test_DELETE_FILEURL_missing2(self):
1180         d = self.DELETE(self.public_url + "/missing/missing")
1181         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1182         return d
1183
1184     def failUnlessHasBarDotTxtMetadata(self, res):
1185         data = simplejson.loads(res)
1186         self.failUnless(isinstance(data, list))
1187         self.failUnlessIn("metadata", data[1])
1188         self.failUnlessIn("tahoe", data[1]["metadata"])
1189         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1190         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1191         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1192                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1193
1194     def test_GET_FILEURL_json(self):
1195         # twisted.web.http.parse_qs ignores any query args without an '=', so
1196         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1197         # instead. This may make it tricky to emulate the S3 interface
1198         # completely.
1199         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1200         def _check1(data):
1201             self.failUnlessIsBarJSON(data)
1202             self.failUnlessHasBarDotTxtMetadata(data)
1203             return
1204         d.addCallback(_check1)
1205         return d
1206
1207     def test_GET_FILEURL_json_mutable_type(self):
1208         # The JSON should include format, which says whether the
1209         # file is SDMF or MDMF
1210         d = self.PUT("/uri?format=mdmf",
1211                      self.NEWFILE_CONTENTS * 300000)
1212         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1213         def _got_json(json, version):
1214             data = simplejson.loads(json)
1215             assert "filenode" == data[0]
1216             data = data[1]
1217             assert isinstance(data, dict)
1218
1219             self.failUnlessIn("format", data)
1220             self.failUnlessEqual(data["format"], version)
1221
1222         d.addCallback(_got_json, "MDMF")
1223         # Now make an SDMF file and check that it is reported correctly.
1224         d.addCallback(lambda ignored:
1225             self.PUT("/uri?format=sdmf",
1226                       self.NEWFILE_CONTENTS * 300000))
1227         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1228         d.addCallback(_got_json, "SDMF")
1229         return d
1230
1231     def test_GET_FILEURL_json_mdmf(self):
1232         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1233         d.addCallback(self.failUnlessIsQuuxJSON)
1234         return d
1235
1236     def test_GET_FILEURL_json_missing(self):
1237         d = self.GET(self.public_url + "/foo/missing?json")
1238         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1239         return d
1240
1241     def test_GET_FILEURL_uri(self):
1242         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1243         def _check(res):
1244             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1245         d.addCallback(_check)
1246         d.addCallback(lambda res:
1247                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1248         def _check2(res):
1249             # for now, for files, uris and readonly-uris are the same
1250             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1251         d.addCallback(_check2)
1252         return d
1253
1254     def test_GET_FILEURL_badtype(self):
1255         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1256                                  "bad t=bogus",
1257                                  self.GET,
1258                                  self.public_url + "/foo/bar.txt?t=bogus")
1259         return d
1260
1261     def test_CSS_FILE(self):
1262         d = self.GET("/tahoe.css", followRedirect=True)
1263         def _check(res):
1264             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1265             self.failUnless(CSS_STYLE.search(res), res)
1266         d.addCallback(_check)
1267         return d
1268
1269     def test_GET_FILEURL_uri_missing(self):
1270         d = self.GET(self.public_url + "/foo/missing?t=uri")
1271         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1272         return d
1273
1274     def _check_upload_and_mkdir_forms(self, html):
1275         # We should have a form to create a file, with radio buttons that allow
1276         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1277         self.failUnlessIn('name="t" value="upload"', html)
1278         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1279         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1280         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1281
1282         # We should also have the ability to create a mutable directory, with
1283         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1284         # or MDMF directory.
1285         self.failUnlessIn('name="t" value="mkdir"', html)
1286         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1287         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1288
1289         self.failUnlessIn(FAVICON_MARKUP, html)
1290
1291     def test_GET_DIRECTORY_html(self):
1292         d = self.GET(self.public_url + "/foo", followRedirect=True)
1293         def _check(html):
1294             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1295             self._check_upload_and_mkdir_forms(html)
1296             self.failUnlessIn("quux", html)
1297         d.addCallback(_check)
1298         return d
1299
1300     def test_GET_root_html(self):
1301         d = self.GET("/")
1302         d.addCallback(self._check_upload_and_mkdir_forms)
1303         return d
1304
1305     def test_GET_DIRURL(self):
1306         # the addSlash means we get a redirect here
1307         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1308         ROOT = "../../.."
1309         d = self.GET(self.public_url + "/foo", followRedirect=True)
1310         def _check(res):
1311             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1312
1313             # the FILE reference points to a URI, but it should end in bar.txt
1314             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1315                        (ROOT, urllib.quote(self._bar_txt_uri)))
1316             get_bar = "".join([r'<td>FILE</td>',
1317                                r'\s+<td>',
1318                                r'<a href="%s">bar.txt</a>' % bar_url,
1319                                r'</td>',
1320                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1321                                ])
1322             self.failUnless(re.search(get_bar, res), res)
1323             for label in ['unlink', 'rename/move']:
1324                 for line in res.split("\n"):
1325                     # find the line that contains the relevant button for bar.txt
1326                     if ("form action" in line and
1327                         ('value="%s"' % (label,)) in line and
1328                         'value="bar.txt"' in line):
1329                         # the form target should use a relative URL
1330                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1331                         self.failUnlessIn('action="%s"' % foo_url, line)
1332                         # and the when_done= should too
1333                         #done_url = urllib.quote(???)
1334                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1335
1336                         # 'unlink' needs to use POST because it directly has a side effect
1337                         if label == 'unlink':
1338                             self.failUnlessIn('method="post"', line)
1339                         break
1340                 else:
1341                     self.fail("unable to find '%s bar.txt' line" % (label,))
1342
1343             # the DIR reference just points to a URI
1344             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1345             get_sub = ((r'<td>DIR</td>')
1346                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1347             self.failUnless(re.search(get_sub, res), res)
1348         d.addCallback(_check)
1349
1350         # look at a readonly directory
1351         d.addCallback(lambda res:
1352                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1353         def _check2(res):
1354             self.failUnlessIn("(read-only)", res)
1355             self.failIfIn("Upload a file", res)
1356         d.addCallback(_check2)
1357
1358         # and at a directory that contains a readonly directory
1359         d.addCallback(lambda res:
1360                       self.GET(self.public_url, followRedirect=True))
1361         def _check3(res):
1362             self.failUnless(re.search('<td>DIR-RO</td>'
1363                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1364         d.addCallback(_check3)
1365
1366         # and an empty directory
1367         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1368         def _check4(res):
1369             self.failUnlessIn("directory is empty", res)
1370             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1371             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1372         d.addCallback(_check4)
1373
1374         # and at a literal directory
1375         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1376         d.addCallback(lambda res:
1377                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1378         def _check5(res):
1379             self.failUnlessIn('(immutable)', res)
1380             self.failUnless(re.search('<td>FILE</td>'
1381                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1382         d.addCallback(_check5)
1383         return d
1384
1385     def test_GET_DIRURL_badtype(self):
1386         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1387                                  400, "Bad Request",
1388                                  "bad t=bogus",
1389                                  self.GET,
1390                                  self.public_url + "/foo?t=bogus")
1391         return d
1392
1393     def test_GET_DIRURL_json(self):
1394         d = self.GET(self.public_url + "/foo?t=json")
1395         d.addCallback(self.failUnlessIsFooJSON)
1396         return d
1397
1398     def test_GET_DIRURL_json_format(self):
1399         d = self.PUT(self.public_url + \
1400                      "/foo/sdmf.txt?format=sdmf",
1401                      self.NEWFILE_CONTENTS * 300000)
1402         d.addCallback(lambda ignored:
1403             self.PUT(self.public_url + \
1404                      "/foo/mdmf.txt?format=mdmf",
1405                      self.NEWFILE_CONTENTS * 300000))
1406         # Now we have an MDMF and SDMF file in the directory. If we GET
1407         # its JSON, we should see their encodings.
1408         d.addCallback(lambda ignored:
1409             self.GET(self.public_url + "/foo?t=json"))
1410         def _got_json(json):
1411             data = simplejson.loads(json)
1412             assert data[0] == "dirnode"
1413
1414             data = data[1]
1415             kids = data['children']
1416
1417             mdmf_data = kids['mdmf.txt'][1]
1418             self.failUnlessIn("format", mdmf_data)
1419             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1420
1421             sdmf_data = kids['sdmf.txt'][1]
1422             self.failUnlessIn("format", sdmf_data)
1423             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1424         d.addCallback(_got_json)
1425         return d
1426
1427
1428     def test_POST_DIRURL_manifest_no_ophandle(self):
1429         d = self.shouldFail2(error.Error,
1430                              "test_POST_DIRURL_manifest_no_ophandle",
1431                              "400 Bad Request",
1432                              "slow operation requires ophandle=",
1433                              self.POST, self.public_url, t="start-manifest")
1434         return d
1435
1436     def test_POST_DIRURL_manifest(self):
1437         d = defer.succeed(None)
1438         def getman(ignored, output):
1439             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1440                           followRedirect=True)
1441             d.addCallback(self.wait_for_operation, "125")
1442             d.addCallback(self.get_operation_results, "125", output)
1443             return d
1444         d.addCallback(getman, None)
1445         def _got_html(manifest):
1446             self.failUnlessIn("Manifest of SI=", manifest)
1447             self.failUnlessIn("<td>sub</td>", manifest)
1448             self.failUnlessIn(self._sub_uri, manifest)
1449             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1450             self.failUnlessIn(FAVICON_MARKUP, manifest)
1451         d.addCallback(_got_html)
1452
1453         # both t=status and unadorned GET should be identical
1454         d.addCallback(lambda res: self.GET("/operations/125"))
1455         d.addCallback(_got_html)
1456
1457         d.addCallback(getman, "html")
1458         d.addCallback(_got_html)
1459         d.addCallback(getman, "text")
1460         def _got_text(manifest):
1461             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1462             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1463         d.addCallback(_got_text)
1464         d.addCallback(getman, "JSON")
1465         def _got_json(res):
1466             data = res["manifest"]
1467             got = {}
1468             for (path_list, cap) in data:
1469                 got[tuple(path_list)] = cap
1470             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1471             self.failUnlessIn((u"sub", u"baz.txt"), got)
1472             self.failUnlessIn("finished", res)
1473             self.failUnlessIn("origin", res)
1474             self.failUnlessIn("storage-index", res)
1475             self.failUnlessIn("verifycaps", res)
1476             self.failUnlessIn("stats", res)
1477         d.addCallback(_got_json)
1478         return d
1479
1480     def test_POST_DIRURL_deepsize_no_ophandle(self):
1481         d = self.shouldFail2(error.Error,
1482                              "test_POST_DIRURL_deepsize_no_ophandle",
1483                              "400 Bad Request",
1484                              "slow operation requires ophandle=",
1485                              self.POST, self.public_url, t="start-deep-size")
1486         return d
1487
1488     def test_POST_DIRURL_deepsize(self):
1489         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1490                       followRedirect=True)
1491         d.addCallback(self.wait_for_operation, "126")
1492         d.addCallback(self.get_operation_results, "126", "json")
1493         def _got_json(data):
1494             self.failUnlessReallyEqual(data["finished"], True)
1495             size = data["size"]
1496             self.failUnless(size > 1000)
1497         d.addCallback(_got_json)
1498         d.addCallback(self.get_operation_results, "126", "text")
1499         def _got_text(res):
1500             mo = re.search(r'^size: (\d+)$', res, re.M)
1501             self.failUnless(mo, res)
1502             size = int(mo.group(1))
1503             # with directories, the size varies.
1504             self.failUnless(size > 1000)
1505         d.addCallback(_got_text)
1506         return d
1507
1508     def test_POST_DIRURL_deepstats_no_ophandle(self):
1509         d = self.shouldFail2(error.Error,
1510                              "test_POST_DIRURL_deepstats_no_ophandle",
1511                              "400 Bad Request",
1512                              "slow operation requires ophandle=",
1513                              self.POST, self.public_url, t="start-deep-stats")
1514         return d
1515
1516     def test_POST_DIRURL_deepstats(self):
1517         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1518                       followRedirect=True)
1519         d.addCallback(self.wait_for_operation, "127")
1520         d.addCallback(self.get_operation_results, "127", "json")
1521         def _got_json(stats):
1522             expected = {"count-immutable-files": 3,
1523                         "count-mutable-files": 2,
1524                         "count-literal-files": 0,
1525                         "count-files": 5,
1526                         "count-directories": 3,
1527                         "size-immutable-files": 57,
1528                         "size-literal-files": 0,
1529                         #"size-directories": 1912, # varies
1530                         #"largest-directory": 1590,
1531                         "largest-directory-children": 7,
1532                         "largest-immutable-file": 19,
1533                         }
1534             for k,v in expected.iteritems():
1535                 self.failUnlessReallyEqual(stats[k], v,
1536                                            "stats[%s] was %s, not %s" %
1537                                            (k, stats[k], v))
1538             self.failUnlessReallyEqual(stats["size-files-histogram"],
1539                                        [ [11, 31, 3] ])
1540         d.addCallback(_got_json)
1541         return d
1542
1543     def test_POST_DIRURL_stream_manifest(self):
1544         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1545         def _check(res):
1546             self.failUnless(res.endswith("\n"))
1547             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1548             self.failUnlessReallyEqual(len(units), 9)
1549             self.failUnlessEqual(units[-1]["type"], "stats")
1550             first = units[0]
1551             self.failUnlessEqual(first["path"], [])
1552             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1553             self.failUnlessEqual(first["type"], "directory")
1554             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1555             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1556             self.failIfEqual(baz["storage-index"], None)
1557             self.failIfEqual(baz["verifycap"], None)
1558             self.failIfEqual(baz["repaircap"], None)
1559             # XXX: Add quux and baz to this test.
1560             return
1561         d.addCallback(_check)
1562         return d
1563
1564     def test_GET_DIRURL_uri(self):
1565         d = self.GET(self.public_url + "/foo?t=uri")
1566         def _check(res):
1567             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1568         d.addCallback(_check)
1569         return d
1570
1571     def test_GET_DIRURL_readonly_uri(self):
1572         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1573         def _check(res):
1574             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1575         d.addCallback(_check)
1576         return d
1577
1578     def test_PUT_NEWDIRURL(self):
1579         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1580         d.addCallback(lambda res:
1581                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1582         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1583         d.addCallback(self.failUnlessNodeKeysAre, [])
1584         return d
1585
1586     def test_PUT_NEWDIRURL_mdmf(self):
1587         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1588         d.addCallback(lambda res:
1589                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1590         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1591         d.addCallback(lambda node:
1592             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1593         return d
1594
1595     def test_PUT_NEWDIRURL_sdmf(self):
1596         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1597                      "")
1598         d.addCallback(lambda res:
1599                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1600         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1601         d.addCallback(lambda node:
1602             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1603         return d
1604
1605     def test_PUT_NEWDIRURL_bad_format(self):
1606         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1607                                     400, "Bad Request", "Unknown format: foo",
1608                                     self.PUT, self.public_url +
1609                                     "/foo/newdir=?t=mkdir&format=foo", "")
1610
1611     def test_POST_NEWDIRURL(self):
1612         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1613         d.addCallback(lambda res:
1614                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1615         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1616         d.addCallback(self.failUnlessNodeKeysAre, [])
1617         return d
1618
1619     def test_POST_NEWDIRURL_mdmf(self):
1620         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1621         d.addCallback(lambda res:
1622                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1623         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1624         d.addCallback(lambda node:
1625             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1626         return d
1627
1628     def test_POST_NEWDIRURL_sdmf(self):
1629         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1630         d.addCallback(lambda res:
1631             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1632         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1633         d.addCallback(lambda node:
1634             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1635         return d
1636
1637     def test_POST_NEWDIRURL_bad_format(self):
1638         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1639                                     400, "Bad Request", "Unknown format: foo",
1640                                     self.POST2, self.public_url + \
1641                                     "/foo/newdir?t=mkdir&format=foo", "")
1642
1643     def test_POST_NEWDIRURL_emptyname(self):
1644         # an empty pathname component (i.e. a double-slash) is disallowed
1645         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1646                              "400 Bad Request",
1647                              "The webapi does not allow empty pathname components, i.e. a double slash",
1648                              self.POST, self.public_url + "//?t=mkdir")
1649         return d
1650
1651     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1652         (newkids, caps) = self._create_initial_children()
1653         query = "/foo/newdir?t=mkdir-with-children"
1654         if version == MDMF_VERSION:
1655             query += "&format=mdmf"
1656         elif version == SDMF_VERSION:
1657             query += "&format=sdmf"
1658         else:
1659             version = SDMF_VERSION # for later
1660         d = self.POST2(self.public_url + query,
1661                        simplejson.dumps(newkids))
1662         def _check(uri):
1663             n = self.s.create_node_from_uri(uri.strip())
1664             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1665             self.failUnlessEqual(n._node.get_version(), version)
1666             d2.addCallback(lambda ign:
1667                            self.failUnlessROChildURIIs(n, u"child-imm",
1668                                                        caps['filecap1']))
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1671                                                        caps['filecap2']))
1672             d2.addCallback(lambda ign:
1673                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1674                                                        caps['filecap3']))
1675             d2.addCallback(lambda ign:
1676                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1677                                                        caps['unknown_rocap']))
1678             d2.addCallback(lambda ign:
1679                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1680                                                        caps['unknown_rwcap']))
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1683                                                        caps['unknown_immcap']))
1684             d2.addCallback(lambda ign:
1685                            self.failUnlessRWChildURIIs(n, u"dirchild",
1686                                                        caps['dircap']))
1687             d2.addCallback(lambda ign:
1688                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1689                                                        caps['litdircap']))
1690             d2.addCallback(lambda ign:
1691                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1692                                                        caps['emptydircap']))
1693             return d2
1694         d.addCallback(_check)
1695         d.addCallback(lambda res:
1696                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1700         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1701         return d
1702
1703     def test_POST_NEWDIRURL_initial_children(self):
1704         return self._do_POST_NEWDIRURL_initial_children_test()
1705
1706     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1707         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1708
1709     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1710         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1711
1712     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1713         (newkids, caps) = self._create_initial_children()
1714         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1715                                     400, "Bad Request", "Unknown format: foo",
1716                                     self.POST2, self.public_url + \
1717                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1718                                     simplejson.dumps(newkids))
1719
1720     def test_POST_NEWDIRURL_immutable(self):
1721         (newkids, caps) = self._create_immutable_children()
1722         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1723                        simplejson.dumps(newkids))
1724         def _check(uri):
1725             n = self.s.create_node_from_uri(uri.strip())
1726             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1727             d2.addCallback(lambda ign:
1728                            self.failUnlessROChildURIIs(n, u"child-imm",
1729                                                        caps['filecap1']))
1730             d2.addCallback(lambda ign:
1731                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1732                                                        caps['unknown_immcap']))
1733             d2.addCallback(lambda ign:
1734                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1735                                                        caps['immdircap']))
1736             d2.addCallback(lambda ign:
1737                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1738                                                        caps['litdircap']))
1739             d2.addCallback(lambda ign:
1740                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1741                                                        caps['emptydircap']))
1742             return d2
1743         d.addCallback(_check)
1744         d.addCallback(lambda res:
1745                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1748         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1749         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1750         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1751         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1753         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1754         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1755         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1756         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1757         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1758         d.addErrback(self.explain_web_error)
1759         return d
1760
1761     def test_POST_NEWDIRURL_immutable_bad(self):
1762         (newkids, caps) = self._create_initial_children()
1763         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1764                              "400 Bad Request",
1765                              "needed to be immutable but was not",
1766                              self.POST2,
1767                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1768                              simplejson.dumps(newkids))
1769         return d
1770
1771     def test_PUT_NEWDIRURL_exists(self):
1772         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1773         d.addCallback(lambda res:
1774                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1775         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1776         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1777         return d
1778
1779     def test_PUT_NEWDIRURL_blocked(self):
1780         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1781                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1782                              self.PUT,
1783                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1784         d.addCallback(lambda res:
1785                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1786         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1787         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1788         return d
1789
1790     def test_PUT_NEWDIRURL_mkdirs(self):
1791         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1792         d.addCallback(lambda res:
1793                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1794         d.addCallback(lambda res:
1795                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1796         d.addCallback(lambda res:
1797                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1798         d.addCallback(self.failUnlessNodeKeysAre, [])
1799         return d
1800
1801     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1802         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1803         d.addCallback(lambda ignored:
1804             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1805         d.addCallback(lambda ignored:
1806             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1807         d.addCallback(lambda ignored:
1808             self._foo_node.get_child_at_path(u"subdir"))
1809         def _got_subdir(subdir):
1810             # XXX: What we want?
1811             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1812             self.failUnlessNodeHasChild(subdir, u"newdir")
1813             return subdir.get_child_at_path(u"newdir")
1814         d.addCallback(_got_subdir)
1815         d.addCallback(lambda newdir:
1816             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1817         return d
1818
1819     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1820         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1821         d.addCallback(lambda ignored:
1822             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1823         d.addCallback(lambda ignored:
1824             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1825         d.addCallback(lambda ignored:
1826             self._foo_node.get_child_at_path(u"subdir"))
1827         def _got_subdir(subdir):
1828             # XXX: What we want?
1829             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1830             self.failUnlessNodeHasChild(subdir, u"newdir")
1831             return subdir.get_child_at_path(u"newdir")
1832         d.addCallback(_got_subdir)
1833         d.addCallback(lambda newdir:
1834             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1838         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1839                                     400, "Bad Request", "Unknown format: foo",
1840                                     self.PUT, self.public_url + \
1841                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1842                                     "")
1843
1844     def test_DELETE_DIRURL(self):
1845         d = self.DELETE(self.public_url + "/foo")
1846         d.addCallback(lambda res:
1847                       self.failIfNodeHasChild(self.public_root, u"foo"))
1848         return d
1849
1850     def test_DELETE_DIRURL_missing(self):
1851         d = self.DELETE(self.public_url + "/foo/missing")
1852         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1853         d.addCallback(lambda res:
1854                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1855         return d
1856
1857     def test_DELETE_DIRURL_missing2(self):
1858         d = self.DELETE(self.public_url + "/missing")
1859         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1860         return d
1861
1862     def dump_root(self):
1863         print "NODEWALK"
1864         w = webish.DirnodeWalkerMixin()
1865         def visitor(childpath, childnode, metadata):
1866             print childpath
1867         d = w.walk(self.public_root, visitor)
1868         return d
1869
1870     def failUnlessNodeKeysAre(self, node, expected_keys):
1871         for k in expected_keys:
1872             assert isinstance(k, unicode)
1873         d = node.list()
1874         def _check(children):
1875             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1876         d.addCallback(_check)
1877         return d
1878     def failUnlessNodeHasChild(self, node, name):
1879         assert isinstance(name, unicode)
1880         d = node.list()
1881         def _check(children):
1882             self.failUnlessIn(name, children)
1883         d.addCallback(_check)
1884         return d
1885     def failIfNodeHasChild(self, node, name):
1886         assert isinstance(name, unicode)
1887         d = node.list()
1888         def _check(children):
1889             self.failIfIn(name, children)
1890         d.addCallback(_check)
1891         return d
1892
1893     def failUnlessChildContentsAre(self, node, name, expected_contents):
1894         assert isinstance(name, unicode)
1895         d = node.get_child_at_path(name)
1896         d.addCallback(lambda node: download_to_data(node))
1897         def _check(contents):
1898             self.failUnlessReallyEqual(contents, expected_contents)
1899         d.addCallback(_check)
1900         return d
1901
1902     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1903         assert isinstance(name, unicode)
1904         d = node.get_child_at_path(name)
1905         d.addCallback(lambda node: node.download_best_version())
1906         def _check(contents):
1907             self.failUnlessReallyEqual(contents, expected_contents)
1908         d.addCallback(_check)
1909         return d
1910
1911     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1912         assert isinstance(name, unicode)
1913         d = node.get_child_at_path(name)
1914         def _check(child):
1915             self.failUnless(child.is_unknown() or not child.is_readonly())
1916             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1917             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1918             expected_ro_uri = self._make_readonly(expected_uri)
1919             if expected_ro_uri:
1920                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1921         d.addCallback(_check)
1922         return d
1923
1924     def failUnlessROChildURIIs(self, node, name, expected_uri):
1925         assert isinstance(name, unicode)
1926         d = node.get_child_at_path(name)
1927         def _check(child):
1928             self.failUnless(child.is_unknown() or child.is_readonly())
1929             self.failUnlessReallyEqual(child.get_write_uri(), None)
1930             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1931             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1932         d.addCallback(_check)
1933         return d
1934
1935     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1936         assert isinstance(name, unicode)
1937         d = node.get_child_at_path(name)
1938         def _check(child):
1939             self.failUnless(child.is_unknown() or not child.is_readonly())
1940             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1941             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1942             expected_ro_uri = self._make_readonly(got_uri)
1943             if expected_ro_uri:
1944                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1945         d.addCallback(_check)
1946         return d
1947
1948     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1949         assert isinstance(name, unicode)
1950         d = node.get_child_at_path(name)
1951         def _check(child):
1952             self.failUnless(child.is_unknown() or child.is_readonly())
1953             self.failUnlessReallyEqual(child.get_write_uri(), None)
1954             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1955             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1956         d.addCallback(_check)
1957         return d
1958
1959     def failUnlessCHKURIHasContents(self, got_uri, contents):
1960         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1961
1962     def test_POST_upload(self):
1963         d = self.POST(self.public_url + "/foo", t="upload",
1964                       file=("new.txt", self.NEWFILE_CONTENTS))
1965         fn = self._foo_node
1966         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1967         d.addCallback(lambda res:
1968                       self.failUnlessChildContentsAre(fn, u"new.txt",
1969                                                       self.NEWFILE_CONTENTS))
1970         return d
1971
1972     def test_POST_upload_unicode(self):
1973         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1974         d = self.POST(self.public_url + "/foo", t="upload",
1975                       file=(filename, self.NEWFILE_CONTENTS))
1976         fn = self._foo_node
1977         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1978         d.addCallback(lambda res:
1979                       self.failUnlessChildContentsAre(fn, filename,
1980                                                       self.NEWFILE_CONTENTS))
1981         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1982         d.addCallback(lambda res: self.GET(target_url))
1983         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1984                                                                   self.NEWFILE_CONTENTS,
1985                                                                   contents))
1986         return d
1987
1988     def test_POST_upload_unicode_named(self):
1989         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1990         d = self.POST(self.public_url + "/foo", t="upload",
1991                       name=filename,
1992                       file=("overridden", self.NEWFILE_CONTENTS))
1993         fn = self._foo_node
1994         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1995         d.addCallback(lambda res:
1996                       self.failUnlessChildContentsAre(fn, filename,
1997                                                       self.NEWFILE_CONTENTS))
1998         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1999         d.addCallback(lambda res: self.GET(target_url))
2000         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2001                                                                   self.NEWFILE_CONTENTS,
2002                                                                   contents))
2003         return d
2004
2005     def test_POST_upload_no_link(self):
2006         d = self.POST("/uri", t="upload",
2007                       file=("new.txt", self.NEWFILE_CONTENTS))
2008         def _check_upload_results(page):
2009             # this should be a page which describes the results of the upload
2010             # that just finished.
2011             self.failUnlessIn("Upload Results:", page)
2012             self.failUnlessIn("URI:", page)
2013             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2014             mo = uri_re.search(page)
2015             self.failUnless(mo, page)
2016             new_uri = mo.group(1)
2017             return new_uri
2018         d.addCallback(_check_upload_results)
2019         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2020         return d
2021
2022     def test_POST_upload_no_link_whendone(self):
2023         d = self.POST("/uri", t="upload", when_done="/",
2024                       file=("new.txt", self.NEWFILE_CONTENTS))
2025         d.addBoth(self.shouldRedirect, "/")
2026         return d
2027
2028     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2029         d = defer.maybeDeferred(callable, *args, **kwargs)
2030         def done(res):
2031             if isinstance(res, failure.Failure):
2032                 res.trap(error.PageRedirect)
2033                 statuscode = res.value.status
2034                 target = res.value.location
2035                 return checker(statuscode, target)
2036             self.fail("%s: callable was supposed to redirect, not return '%s'"
2037                       % (which, res))
2038         d.addBoth(done)
2039         return d
2040
2041     def test_POST_upload_no_link_whendone_results(self):
2042         def check(statuscode, target):
2043             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2044             self.failUnless(target.startswith(self.webish_url), target)
2045             return client.getPage(target, method="GET")
2046         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2047                                  check,
2048                                  self.POST, "/uri", t="upload",
2049                                  when_done="/uri/%(uri)s",
2050                                  file=("new.txt", self.NEWFILE_CONTENTS))
2051         d.addCallback(lambda res:
2052                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2053         return d
2054
2055     def test_POST_upload_no_link_mutable(self):
2056         d = self.POST("/uri", t="upload", mutable="true",
2057                       file=("new.txt", self.NEWFILE_CONTENTS))
2058         def _check(filecap):
2059             filecap = filecap.strip()
2060             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2061             self.filecap = filecap
2062             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2063             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2064             n = self.s.create_node_from_uri(filecap)
2065             return n.download_best_version()
2066         d.addCallback(_check)
2067         def _check2(data):
2068             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2069             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2070         d.addCallback(_check2)
2071         def _check3(data):
2072             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2073             return self.GET("/file/%s" % urllib.quote(self.filecap))
2074         d.addCallback(_check3)
2075         def _check4(data):
2076             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2077         d.addCallback(_check4)
2078         return d
2079
2080     def test_POST_upload_no_link_mutable_toobig(self):
2081         # The SDMF size limit is no longer in place, so we should be
2082         # able to upload mutable files that are as large as we want them
2083         # to be.
2084         d = self.POST("/uri", t="upload", mutable="true",
2085                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2086         return d
2087
2088
2089     def test_POST_upload_format_unlinked(self):
2090         def _check_upload_unlinked(ign, format, uri_prefix):
2091             filename = format + ".txt"
2092             d = self.POST("/uri?t=upload&format=" + format,
2093                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2094             def _got_results(results):
2095                 if format.upper() in ("SDMF", "MDMF"):
2096                     # webapi.rst says this returns a filecap
2097                     filecap = results
2098                 else:
2099                     # for immutable, it returns an "upload results page", and
2100                     # the filecap is buried inside
2101                     line = [l for l in results.split("\n") if "URI: " in l][0]
2102                     mo = re.search(r'<span>([^<]+)</span>', line)
2103                     filecap = mo.group(1)
2104                 self.failUnless(filecap.startswith(uri_prefix),
2105                                 (uri_prefix, filecap))
2106                 return self.GET("/uri/%s?t=json" % filecap)
2107             d.addCallback(_got_results)
2108             def _got_json(json):
2109                 data = simplejson.loads(json)
2110                 data = data[1]
2111                 self.failUnlessIn("format", data)
2112                 self.failUnlessEqual(data["format"], format.upper())
2113             d.addCallback(_got_json)
2114             return d
2115         d = defer.succeed(None)
2116         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2117         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2118         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2119         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2120         return d
2121
2122     def test_POST_upload_bad_format_unlinked(self):
2123         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2124                                     400, "Bad Request", "Unknown format: foo",
2125                                     self.POST,
2126                                     "/uri?t=upload&format=foo",
2127                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2128
2129     def test_POST_upload_format(self):
2130         def _check_upload(ign, format, uri_prefix, fn=None):
2131             filename = format + ".txt"
2132             d = self.POST(self.public_url +
2133                           "/foo?t=upload&format=" + format,
2134                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2135             def _got_filecap(filecap):
2136                 if fn is not None:
2137                     filenameu = unicode(filename)
2138                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2139                 self.failUnless(filecap.startswith(uri_prefix))
2140                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2141             d.addCallback(_got_filecap)
2142             def _got_json(json):
2143                 data = simplejson.loads(json)
2144                 data = data[1]
2145                 self.failUnlessIn("format", data)
2146                 self.failUnlessEqual(data["format"], format.upper())
2147             d.addCallback(_got_json)
2148             return d
2149
2150         d = defer.succeed(None)
2151         d.addCallback(_check_upload, "chk", "URI:CHK")
2152         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2153         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2154         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2155         return d
2156
2157     def test_POST_upload_bad_format(self):
2158         return self.shouldHTTPError("POST_upload_bad_format",
2159                                     400, "Bad Request", "Unknown format: foo",
2160                                     self.POST, self.public_url + \
2161                                     "/foo?t=upload&format=foo",
2162                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2163
2164     def test_POST_upload_mutable(self):
2165         # this creates a mutable file
2166         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2167                       file=("new.txt", self.NEWFILE_CONTENTS))
2168         fn = self._foo_node
2169         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2170         d.addCallback(lambda res:
2171                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2172                                                              self.NEWFILE_CONTENTS))
2173         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2174         def _got(newnode):
2175             self.failUnless(IMutableFileNode.providedBy(newnode))
2176             self.failUnless(newnode.is_mutable())
2177             self.failIf(newnode.is_readonly())
2178             self._mutable_node = newnode
2179             self._mutable_uri = newnode.get_uri()
2180         d.addCallback(_got)
2181
2182         # now upload it again and make sure that the URI doesn't change
2183         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2184         d.addCallback(lambda res:
2185                       self.POST(self.public_url + "/foo", t="upload",
2186                                 mutable="true",
2187                                 file=("new.txt", NEWER_CONTENTS)))
2188         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2189         d.addCallback(lambda res:
2190                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2191                                                              NEWER_CONTENTS))
2192         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2193         def _got2(newnode):
2194             self.failUnless(IMutableFileNode.providedBy(newnode))
2195             self.failUnless(newnode.is_mutable())
2196             self.failIf(newnode.is_readonly())
2197             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2198         d.addCallback(_got2)
2199
2200         # upload a second time, using PUT instead of POST
2201         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2202         d.addCallback(lambda res:
2203                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2204         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2205         d.addCallback(lambda res:
2206                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2207                                                              NEW2_CONTENTS))
2208
2209         # finally list the directory, since mutable files are displayed
2210         # slightly differently
2211
2212         d.addCallback(lambda res:
2213                       self.GET(self.public_url + "/foo/",
2214                                followRedirect=True))
2215         def _check_page(res):
2216             # TODO: assert more about the contents
2217             self.failUnlessIn("SSK", res)
2218             return res
2219         d.addCallback(_check_page)
2220
2221         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2222         def _got3(newnode):
2223             self.failUnless(IMutableFileNode.providedBy(newnode))
2224             self.failUnless(newnode.is_mutable())
2225             self.failIf(newnode.is_readonly())
2226             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2227         d.addCallback(_got3)
2228
2229         # look at the JSON form of the enclosing directory
2230         d.addCallback(lambda res:
2231                       self.GET(self.public_url + "/foo/?t=json",
2232                                followRedirect=True))
2233         def _check_page_json(res):
2234             parsed = simplejson.loads(res)
2235             self.failUnlessEqual(parsed[0], "dirnode")
2236             children = dict( [(unicode(name),value)
2237                               for (name,value)
2238                               in parsed[1]["children"].iteritems()] )
2239             self.failUnlessIn(u"new.txt", children)
2240             new_json = children[u"new.txt"]
2241             self.failUnlessEqual(new_json[0], "filenode")
2242             self.failUnless(new_json[1]["mutable"])
2243             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2244             ro_uri = self._mutable_node.get_readonly().to_string()
2245             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2246         d.addCallback(_check_page_json)
2247
2248         # and the JSON form of the file
2249         d.addCallback(lambda res:
2250                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2251         def _check_file_json(res):
2252             parsed = simplejson.loads(res)
2253             self.failUnlessEqual(parsed[0], "filenode")
2254             self.failUnless(parsed[1]["mutable"])
2255             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2256             ro_uri = self._mutable_node.get_readonly().to_string()
2257             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2258         d.addCallback(_check_file_json)
2259
2260         # and look at t=uri and t=readonly-uri
2261         d.addCallback(lambda res:
2262                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2263         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2264         d.addCallback(lambda res:
2265                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2266         def _check_ro_uri(res):
2267             ro_uri = self._mutable_node.get_readonly().to_string()
2268             self.failUnlessReallyEqual(res, ro_uri)
2269         d.addCallback(_check_ro_uri)
2270
2271         # make sure we can get to it from /uri/URI
2272         d.addCallback(lambda res:
2273                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2274         d.addCallback(lambda res:
2275                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2276
2277         # and that HEAD computes the size correctly
2278         d.addCallback(lambda res:
2279                       self.HEAD(self.public_url + "/foo/new.txt",
2280                                 return_response=True))
2281         def _got_headers((res, status, headers)):
2282             self.failUnlessReallyEqual(res, "")
2283             self.failUnlessReallyEqual(headers["content-length"][0],
2284                                        str(len(NEW2_CONTENTS)))
2285             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2286         d.addCallback(_got_headers)
2287
2288         # make sure that outdated size limits aren't enforced anymore.
2289         d.addCallback(lambda ignored:
2290             self.POST(self.public_url + "/foo", t="upload",
2291                       mutable="true",
2292                       file=("new.txt",
2293                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2294         d.addErrback(self.dump_error)
2295         return d
2296
2297     def test_POST_upload_mutable_toobig(self):
2298         # SDMF had a size limti that was removed a while ago. MDMF has
2299         # never had a size limit. Test to make sure that we do not
2300         # encounter errors when trying to upload large mutable files,
2301         # since there should be no coded prohibitions regarding large
2302         # mutable files.
2303         d = self.POST(self.public_url + "/foo",
2304                       t="upload", mutable="true",
2305                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2306         return d
2307
2308     def dump_error(self, f):
2309         # if the web server returns an error code (like 400 Bad Request),
2310         # web.client.getPage puts the HTTP response body into the .response
2311         # attribute of the exception object that it gives back. It does not
2312         # appear in the Failure's repr(), so the ERROR that trial displays
2313         # will be rather terse and unhelpful. addErrback this method to the
2314         # end of your chain to get more information out of these errors.
2315         if f.check(error.Error):
2316             print "web.error.Error:"
2317             print f
2318             print f.value.response
2319         return f
2320
2321     def test_POST_upload_replace(self):
2322         d = self.POST(self.public_url + "/foo", t="upload",
2323                       file=("bar.txt", self.NEWFILE_CONTENTS))
2324         fn = self._foo_node
2325         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2326         d.addCallback(lambda res:
2327                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2328                                                       self.NEWFILE_CONTENTS))
2329         return d
2330
2331     def test_POST_upload_no_replace_ok(self):
2332         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2333                       file=("new.txt", self.NEWFILE_CONTENTS))
2334         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2335         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2336                                                              self.NEWFILE_CONTENTS))
2337         return d
2338
2339     def test_POST_upload_no_replace_queryarg(self):
2340         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2341                       file=("bar.txt", self.NEWFILE_CONTENTS))
2342         d.addBoth(self.shouldFail, error.Error,
2343                   "POST_upload_no_replace_queryarg",
2344                   "409 Conflict",
2345                   "There was already a child by that name, and you asked me "
2346                   "to not replace it")
2347         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2348         d.addCallback(self.failUnlessIsBarDotTxt)
2349         return d
2350
2351     def test_POST_upload_no_replace_field(self):
2352         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2353                       file=("bar.txt", self.NEWFILE_CONTENTS))
2354         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2355                   "409 Conflict",
2356                   "There was already a child by that name, and you asked me "
2357                   "to not replace it")
2358         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2359         d.addCallback(self.failUnlessIsBarDotTxt)
2360         return d
2361
2362     def test_POST_upload_whendone(self):
2363         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2364                       file=("new.txt", self.NEWFILE_CONTENTS))
2365         d.addBoth(self.shouldRedirect, "/THERE")
2366         fn = self._foo_node
2367         d.addCallback(lambda res:
2368                       self.failUnlessChildContentsAre(fn, u"new.txt",
2369                                                       self.NEWFILE_CONTENTS))
2370         return d
2371
2372     def test_POST_upload_named(self):
2373         fn = self._foo_node
2374         d = self.POST(self.public_url + "/foo", t="upload",
2375                       name="new.txt", file=self.NEWFILE_CONTENTS)
2376         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2377         d.addCallback(lambda res:
2378                       self.failUnlessChildContentsAre(fn, u"new.txt",
2379                                                       self.NEWFILE_CONTENTS))
2380         return d
2381
2382     def test_POST_upload_named_badfilename(self):
2383         d = self.POST(self.public_url + "/foo", t="upload",
2384                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2385         d.addBoth(self.shouldFail, error.Error,
2386                   "test_POST_upload_named_badfilename",
2387                   "400 Bad Request",
2388                   "name= may not contain a slash",
2389                   )
2390         # make sure that nothing was added
2391         d.addCallback(lambda res:
2392                       self.failUnlessNodeKeysAre(self._foo_node,
2393                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2394                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2395                                                   u"sub"]))
2396         return d
2397
2398     def test_POST_FILEURL_check(self):
2399         bar_url = self.public_url + "/foo/bar.txt"
2400         d = self.POST(bar_url, t="check")
2401         def _check(res):
2402             self.failUnlessIn("Healthy :", res)
2403         d.addCallback(_check)
2404         redir_url = "http://allmydata.org/TARGET"
2405         def _check2(statuscode, target):
2406             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2407             self.failUnlessReallyEqual(target, redir_url)
2408         d.addCallback(lambda res:
2409                       self.shouldRedirect2("test_POST_FILEURL_check",
2410                                            _check2,
2411                                            self.POST, bar_url,
2412                                            t="check",
2413                                            when_done=redir_url))
2414         d.addCallback(lambda res:
2415                       self.POST(bar_url, t="check", return_to=redir_url))
2416         def _check3(res):
2417             self.failUnlessIn("Healthy :", res)
2418             self.failUnlessIn("Return to file", res)
2419             self.failUnlessIn(redir_url, res)
2420         d.addCallback(_check3)
2421
2422         d.addCallback(lambda res:
2423                       self.POST(bar_url, t="check", output="JSON"))
2424         def _check_json(res):
2425             data = simplejson.loads(res)
2426             self.failUnlessIn("storage-index", data)
2427             self.failUnless(data["results"]["healthy"])
2428         d.addCallback(_check_json)
2429
2430         return d
2431
2432     def test_POST_FILEURL_check_and_repair(self):
2433         bar_url = self.public_url + "/foo/bar.txt"
2434         d = self.POST(bar_url, t="check", repair="true")
2435         def _check(res):
2436             self.failUnlessIn("Healthy :", res)
2437         d.addCallback(_check)
2438         redir_url = "http://allmydata.org/TARGET"
2439         def _check2(statuscode, target):
2440             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2441             self.failUnlessReallyEqual(target, redir_url)
2442         d.addCallback(lambda res:
2443                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2444                                            _check2,
2445                                            self.POST, bar_url,
2446                                            t="check", repair="true",
2447                                            when_done=redir_url))
2448         d.addCallback(lambda res:
2449                       self.POST(bar_url, t="check", return_to=redir_url))
2450         def _check3(res):
2451             self.failUnlessIn("Healthy :", res)
2452             self.failUnlessIn("Return to file", res)
2453             self.failUnlessIn(redir_url, res)
2454         d.addCallback(_check3)
2455         return d
2456
2457     def test_POST_DIRURL_check(self):
2458         foo_url = self.public_url + "/foo/"
2459         d = self.POST(foo_url, t="check")
2460         def _check(res):
2461             self.failUnlessIn("Healthy :", res)
2462         d.addCallback(_check)
2463         redir_url = "http://allmydata.org/TARGET"
2464         def _check2(statuscode, target):
2465             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2466             self.failUnlessReallyEqual(target, redir_url)
2467         d.addCallback(lambda res:
2468                       self.shouldRedirect2("test_POST_DIRURL_check",
2469                                            _check2,
2470                                            self.POST, foo_url,
2471                                            t="check",
2472                                            when_done=redir_url))
2473         d.addCallback(lambda res:
2474                       self.POST(foo_url, t="check", return_to=redir_url))
2475         def _check3(res):
2476             self.failUnlessIn("Healthy :", res)
2477             self.failUnlessIn("Return to file/directory", res)
2478             self.failUnlessIn(redir_url, res)
2479         d.addCallback(_check3)
2480
2481         d.addCallback(lambda res:
2482                       self.POST(foo_url, t="check", output="JSON"))
2483         def _check_json(res):
2484             data = simplejson.loads(res)
2485             self.failUnlessIn("storage-index", data)
2486             self.failUnless(data["results"]["healthy"])
2487         d.addCallback(_check_json)
2488
2489         return d
2490
2491     def test_POST_DIRURL_check_and_repair(self):
2492         foo_url = self.public_url + "/foo/"
2493         d = self.POST(foo_url, t="check", repair="true")
2494         def _check(res):
2495             self.failUnlessIn("Healthy :", res)
2496         d.addCallback(_check)
2497         redir_url = "http://allmydata.org/TARGET"
2498         def _check2(statuscode, target):
2499             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2500             self.failUnlessReallyEqual(target, redir_url)
2501         d.addCallback(lambda res:
2502                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2503                                            _check2,
2504                                            self.POST, foo_url,
2505                                            t="check", repair="true",
2506                                            when_done=redir_url))
2507         d.addCallback(lambda res:
2508                       self.POST(foo_url, t="check", return_to=redir_url))
2509         def _check3(res):
2510             self.failUnlessIn("Healthy :", res)
2511             self.failUnlessIn("Return to file/directory", res)
2512             self.failUnlessIn(redir_url, res)
2513         d.addCallback(_check3)
2514         return d
2515
2516     def test_POST_FILEURL_mdmf_check(self):
2517         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2518         d = self.POST(quux_url, t="check")
2519         def _check(res):
2520             self.failUnlessIn("Healthy", res)
2521         d.addCallback(_check)
2522         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2523         d.addCallback(lambda ignored:
2524                       self.POST(quux_extension_url, t="check"))
2525         d.addCallback(_check)
2526         return d
2527
2528     def test_POST_FILEURL_mdmf_check_and_repair(self):
2529         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2530         d = self.POST(quux_url, t="check", repair="true")
2531         def _check(res):
2532             self.failUnlessIn("Healthy", res)
2533         d.addCallback(_check)
2534         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2535         d.addCallback(lambda ignored:
2536                       self.POST(quux_extension_url, t="check", repair="true"))
2537         d.addCallback(_check)
2538         return d
2539
2540     def wait_for_operation(self, ignored, ophandle):
2541         url = "/operations/" + ophandle
2542         url += "?t=status&output=JSON"
2543         d = self.GET(url)
2544         def _got(res):
2545             data = simplejson.loads(res)
2546             if not data["finished"]:
2547                 d = self.stall(delay=1.0)
2548                 d.addCallback(self.wait_for_operation, ophandle)
2549                 return d
2550             return data
2551         d.addCallback(_got)
2552         return d
2553
2554     def get_operation_results(self, ignored, ophandle, output=None):
2555         url = "/operations/" + ophandle
2556         url += "?t=status"
2557         if output:
2558             url += "&output=" + output
2559         d = self.GET(url)
2560         def _got(res):
2561             if output and output.lower() == "json":
2562                 return simplejson.loads(res)
2563             return res
2564         d.addCallback(_got)
2565         return d
2566
2567     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2568         d = self.shouldFail2(error.Error,
2569                              "test_POST_DIRURL_deepcheck_no_ophandle",
2570                              "400 Bad Request",
2571                              "slow operation requires ophandle=",
2572                              self.POST, self.public_url, t="start-deep-check")
2573         return d
2574
2575     def test_POST_DIRURL_deepcheck(self):
2576         def _check_redirect(statuscode, target):
2577             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2578             self.failUnless(target.endswith("/operations/123"))
2579         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2580                                  self.POST, self.public_url,
2581                                  t="start-deep-check", ophandle="123")
2582         d.addCallback(self.wait_for_operation, "123")
2583         def _check_json(data):
2584             self.failUnlessReallyEqual(data["finished"], True)
2585             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2586             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2587         d.addCallback(_check_json)
2588         d.addCallback(self.get_operation_results, "123", "html")
2589         def _check_html(res):
2590             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2591             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2592             self.failUnlessIn(FAVICON_MARKUP, res)
2593         d.addCallback(_check_html)
2594
2595         d.addCallback(lambda res:
2596                       self.GET("/operations/123/"))
2597         d.addCallback(_check_html) # should be the same as without the slash
2598
2599         d.addCallback(lambda res:
2600                       self.shouldFail2(error.Error, "one", "404 Not Found",
2601                                        "No detailed results for SI bogus",
2602                                        self.GET, "/operations/123/bogus"))
2603
2604         foo_si = self._foo_node.get_storage_index()
2605         foo_si_s = base32.b2a(foo_si)
2606         d.addCallback(lambda res:
2607                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2608         def _check_foo_json(res):
2609             data = simplejson.loads(res)
2610             self.failUnlessEqual(data["storage-index"], foo_si_s)
2611             self.failUnless(data["results"]["healthy"])
2612         d.addCallback(_check_foo_json)
2613         return d
2614
2615     def test_POST_DIRURL_deepcheck_and_repair(self):
2616         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2617                       ophandle="124", output="json", followRedirect=True)
2618         d.addCallback(self.wait_for_operation, "124")
2619         def _check_json(data):
2620             self.failUnlessReallyEqual(data["finished"], True)
2621             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2622             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2623             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2624             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2625             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2626             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2627             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2628             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2629             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2630             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2631         d.addCallback(_check_json)
2632         d.addCallback(self.get_operation_results, "124", "html")
2633         def _check_html(res):
2634             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2635
2636             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2637             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2638             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2639
2640             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2641             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2642             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2643
2644             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2645             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2646             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2647
2648             self.failUnlessIn(FAVICON_MARKUP, res)
2649         d.addCallback(_check_html)
2650         return d
2651
2652     def test_POST_FILEURL_bad_t(self):
2653         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2654                              "POST to file: bad t=bogus",
2655                              self.POST, self.public_url + "/foo/bar.txt",
2656                              t="bogus")
2657         return d
2658
2659     def test_POST_mkdir(self): # return value?
2660         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2661         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2662         d.addCallback(self.failUnlessNodeKeysAre, [])
2663         return d
2664
2665     def test_POST_mkdir_mdmf(self):
2666         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2667         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2668         d.addCallback(lambda node:
2669             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2670         return d
2671
2672     def test_POST_mkdir_sdmf(self):
2673         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2675         d.addCallback(lambda node:
2676             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2677         return d
2678
2679     def test_POST_mkdir_bad_format(self):
2680         return self.shouldHTTPError("POST_mkdir_bad_format",
2681                                     400, "Bad Request", "Unknown format: foo",
2682                                     self.POST, self.public_url +
2683                                     "/foo?t=mkdir&name=newdir&format=foo")
2684
2685     def test_POST_mkdir_initial_children(self):
2686         (newkids, caps) = self._create_initial_children()
2687         d = self.POST2(self.public_url +
2688                        "/foo?t=mkdir-with-children&name=newdir",
2689                        simplejson.dumps(newkids))
2690         d.addCallback(lambda res:
2691                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2693         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2695         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2696         return d
2697
2698     def test_POST_mkdir_initial_children_mdmf(self):
2699         (newkids, caps) = self._create_initial_children()
2700         d = self.POST2(self.public_url +
2701                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2702                        simplejson.dumps(newkids))
2703         d.addCallback(lambda res:
2704                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(lambda node:
2707             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2708         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2709         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2710                        caps['filecap1'])
2711         return d
2712
2713     # XXX: Duplication.
2714     def test_POST_mkdir_initial_children_sdmf(self):
2715         (newkids, caps) = self._create_initial_children()
2716         d = self.POST2(self.public_url +
2717                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2718                        simplejson.dumps(newkids))
2719         d.addCallback(lambda res:
2720                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2721         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2722         d.addCallback(lambda node:
2723             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2724         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2725         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2726                        caps['filecap1'])
2727         return d
2728
2729     def test_POST_mkdir_initial_children_bad_format(self):
2730         (newkids, caps) = self._create_initial_children()
2731         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2732                                     400, "Bad Request", "Unknown format: foo",
2733                                     self.POST, self.public_url + \
2734                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2735                                     simplejson.dumps(newkids))
2736
2737     def test_POST_mkdir_immutable(self):
2738         (newkids, caps) = self._create_immutable_children()
2739         d = self.POST2(self.public_url +
2740                        "/foo?t=mkdir-immutable&name=newdir",
2741                        simplejson.dumps(newkids))
2742         d.addCallback(lambda res:
2743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2745         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2747         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2748         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2749         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2750         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2751         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2754         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2755         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2756         return d
2757
2758     def test_POST_mkdir_immutable_bad(self):
2759         (newkids, caps) = self._create_initial_children()
2760         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2761                              "400 Bad Request",
2762                              "needed to be immutable but was not",
2763                              self.POST2,
2764                              self.public_url +
2765                              "/foo?t=mkdir-immutable&name=newdir",
2766                              simplejson.dumps(newkids))
2767         return d
2768
2769     def test_POST_mkdir_2(self):
2770         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2771         d.addCallback(lambda res:
2772                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2773         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2774         d.addCallback(self.failUnlessNodeKeysAre, [])
2775         return d
2776
2777     def test_POST_mkdirs_2(self):
2778         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2779         d.addCallback(lambda res:
2780                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2781         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2782         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2783         d.addCallback(self.failUnlessNodeKeysAre, [])
2784         return d
2785
2786     def test_POST_mkdir_no_parentdir_noredirect(self):
2787         d = self.POST("/uri?t=mkdir")
2788         def _after_mkdir(res):
2789             uri.DirectoryURI.init_from_string(res)
2790         d.addCallback(_after_mkdir)
2791         return d
2792
2793     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2794         d = self.POST("/uri?t=mkdir&format=mdmf")
2795         def _after_mkdir(res):
2796             u = uri.from_string(res)
2797             # Check that this is an MDMF writecap
2798             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2799         d.addCallback(_after_mkdir)
2800         return d
2801
2802     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2803         d = self.POST("/uri?t=mkdir&format=sdmf")
2804         def _after_mkdir(res):
2805             u = uri.from_string(res)
2806             self.failUnlessIsInstance(u, uri.DirectoryURI)
2807         d.addCallback(_after_mkdir)
2808         return d
2809
2810     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2811         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2812                                     400, "Bad Request", "Unknown format: foo",
2813                                     self.POST, self.public_url +
2814                                     "/uri?t=mkdir&format=foo")
2815
2816     def test_POST_mkdir_no_parentdir_noredirect2(self):
2817         # make sure form-based arguments (as on the welcome page) still work
2818         d = self.POST("/uri", t="mkdir")
2819         def _after_mkdir(res):
2820             uri.DirectoryURI.init_from_string(res)
2821         d.addCallback(_after_mkdir)
2822         d.addErrback(self.explain_web_error)
2823         return d
2824
2825     def test_POST_mkdir_no_parentdir_redirect(self):
2826         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2827         d.addBoth(self.shouldRedirect, None, statuscode='303')
2828         def _check_target(target):
2829             target = urllib.unquote(target)
2830             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2831         d.addCallback(_check_target)
2832         return d
2833
2834     def test_POST_mkdir_no_parentdir_redirect2(self):
2835         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2836         d.addBoth(self.shouldRedirect, None, statuscode='303')
2837         def _check_target(target):
2838             target = urllib.unquote(target)
2839             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2840         d.addCallback(_check_target)
2841         d.addErrback(self.explain_web_error)
2842         return d
2843
2844     def _make_readonly(self, u):
2845         ro_uri = uri.from_string(u).get_readonly()
2846         if ro_uri is None:
2847             return None
2848         return ro_uri.to_string()
2849
2850     def _create_initial_children(self):
2851         contents, n, filecap1 = self.makefile(12)
2852         md1 = {"metakey1": "metavalue1"}
2853         filecap2 = make_mutable_file_uri()
2854         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2855         filecap3 = node3.get_readonly_uri()
2856         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2857         dircap = DirectoryNode(node4, None, None).get_uri()
2858         mdmfcap = make_mutable_file_uri(mdmf=True)
2859         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2860         emptydircap = "URI:DIR2-LIT:"
2861         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2862                                                       "ro_uri": self._make_readonly(filecap1),
2863                                                       "metadata": md1, }],
2864                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2865                                                       "ro_uri": self._make_readonly(filecap2)}],
2866                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2867                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2868                                                       "ro_uri": unknown_rocap}],
2869                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2870                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2871                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2872                                                       "ro_uri": self._make_readonly(dircap)}],
2873                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2874                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2875                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2876                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2877                    }
2878         return newkids, {'filecap1': filecap1,
2879                          'filecap2': filecap2,
2880                          'filecap3': filecap3,
2881                          'unknown_rwcap': unknown_rwcap,
2882                          'unknown_rocap': unknown_rocap,
2883                          'unknown_immcap': unknown_immcap,
2884                          'dircap': dircap,
2885                          'litdircap': litdircap,
2886                          'emptydircap': emptydircap,
2887                          'mdmfcap': mdmfcap}
2888
2889     def _create_immutable_children(self):
2890         contents, n, filecap1 = self.makefile(12)
2891         md1 = {"metakey1": "metavalue1"}
2892         tnode = create_chk_filenode("immutable directory contents\n"*10)
2893         dnode = DirectoryNode(tnode, None, None)
2894         assert not dnode.is_mutable()
2895         immdircap = dnode.get_uri()
2896         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2897         emptydircap = "URI:DIR2-LIT:"
2898         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2899                                                       "metadata": md1, }],
2900                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2901                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2902                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2903                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2904                    }
2905         return newkids, {'filecap1': filecap1,
2906                          'unknown_immcap': unknown_immcap,
2907                          'immdircap': immdircap,
2908                          'litdircap': litdircap,
2909                          'emptydircap': emptydircap}
2910
2911     def test_POST_mkdir_no_parentdir_initial_children(self):
2912         (newkids, caps) = self._create_initial_children()
2913         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2914         def _after_mkdir(res):
2915             self.failUnless(res.startswith("URI:DIR"), res)
2916             n = self.s.create_node_from_uri(res)
2917             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2918             d2.addCallback(lambda ign:
2919                            self.failUnlessROChildURIIs(n, u"child-imm",
2920                                                        caps['filecap1']))
2921             d2.addCallback(lambda ign:
2922                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2923                                                        caps['filecap2']))
2924             d2.addCallback(lambda ign:
2925                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2926                                                        caps['filecap3']))
2927             d2.addCallback(lambda ign:
2928                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2929                                                        caps['unknown_rwcap']))
2930             d2.addCallback(lambda ign:
2931                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2932                                                        caps['unknown_rocap']))
2933             d2.addCallback(lambda ign:
2934                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2935                                                        caps['unknown_immcap']))
2936             d2.addCallback(lambda ign:
2937                            self.failUnlessRWChildURIIs(n, u"dirchild",
2938                                                        caps['dircap']))
2939             return d2
2940         d.addCallback(_after_mkdir)
2941         return d
2942
2943     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2944         # the regular /uri?t=mkdir operation is specified to ignore its body.
2945         # Only t=mkdir-with-children pays attention to it.
2946         (newkids, caps) = self._create_initial_children()
2947         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2948                                  400, "Bad Request",
2949                                  "t=mkdir does not accept children=, "
2950                                  "try t=mkdir-with-children instead",
2951                                  self.POST2, "/uri?t=mkdir", # without children
2952                                  simplejson.dumps(newkids))
2953         return d
2954
2955     def test_POST_noparent_bad(self):
2956         d = self.shouldHTTPError("POST_noparent_bad",
2957                                  400, "Bad Request",
2958                                  "/uri accepts only PUT, PUT?t=mkdir, "
2959                                  "POST?t=upload, and POST?t=mkdir",
2960                                  self.POST, "/uri?t=bogus")
2961         return d
2962
2963     def test_POST_mkdir_no_parentdir_immutable(self):
2964         (newkids, caps) = self._create_immutable_children()
2965         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2966         def _after_mkdir(res):
2967             self.failUnless(res.startswith("URI:DIR"), res)
2968             n = self.s.create_node_from_uri(res)
2969             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2970             d2.addCallback(lambda ign:
2971                            self.failUnlessROChildURIIs(n, u"child-imm",
2972                                                           caps['filecap1']))
2973             d2.addCallback(lambda ign:
2974                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2975                                                           caps['unknown_immcap']))
2976             d2.addCallback(lambda ign:
2977                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2978                                                           caps['immdircap']))
2979             d2.addCallback(lambda ign:
2980                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2981                                                           caps['litdircap']))
2982             d2.addCallback(lambda ign:
2983                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2984                                                           caps['emptydircap']))
2985             return d2
2986         d.addCallback(_after_mkdir)
2987         return d
2988
2989     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2990         (newkids, caps) = self._create_initial_children()
2991         d = self.shouldFail2(error.Error,
2992                              "test_POST_mkdir_no_parentdir_immutable_bad",
2993                              "400 Bad Request",
2994                              "needed to be immutable but was not",
2995                              self.POST2,
2996                              "/uri?t=mkdir-immutable",
2997                              simplejson.dumps(newkids))
2998         return d
2999
3000     def test_welcome_page_mkdir_button(self):
3001         # Fetch the welcome page.
3002         d = self.GET("/")
3003         def _after_get_welcome_page(res):
3004             MKDIR_BUTTON_RE = re.compile(
3005                 '<form action="([^"]*)" method="post".*?'
3006                 '<input type="hidden" name="t" value="([^"]*)" />'
3007                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3008                 '<input type="submit" value="Create a directory" />',
3009                 re.I)
3010             mo = MKDIR_BUTTON_RE.search(res)
3011             formaction = mo.group(1)
3012             formt = mo.group(2)
3013             formaname = mo.group(3)
3014             formavalue = mo.group(4)
3015             return (formaction, formt, formaname, formavalue)
3016         d.addCallback(_after_get_welcome_page)
3017         def _after_parse_form(res):
3018             (formaction, formt, formaname, formavalue) = res
3019             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3020         d.addCallback(_after_parse_form)
3021         d.addBoth(self.shouldRedirect, None, statuscode='303')
3022         return d
3023
3024     def test_POST_mkdir_replace(self): # return value?
3025         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3026         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3027         d.addCallback(self.failUnlessNodeKeysAre, [])
3028         return d
3029
3030     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3031         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3032         d.addBoth(self.shouldFail, error.Error,
3033                   "POST_mkdir_no_replace_queryarg",
3034                   "409 Conflict",
3035                   "There was already a child by that name, and you asked me "
3036                   "to not replace it")
3037         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3038         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3039         return d
3040
3041     def test_POST_mkdir_no_replace_field(self): # return value?
3042         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3043                       replace="false")
3044         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3045                   "409 Conflict",
3046                   "There was already a child by that name, and you asked me "
3047                   "to not replace it")
3048         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3049         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3050         return d
3051
3052     def test_POST_mkdir_whendone_field(self):
3053         d = self.POST(self.public_url + "/foo",
3054                       t="mkdir", name="newdir", when_done="/THERE")
3055         d.addBoth(self.shouldRedirect, "/THERE")
3056         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3057         d.addCallback(self.failUnlessNodeKeysAre, [])
3058         return d
3059
3060     def test_POST_mkdir_whendone_queryarg(self):
3061         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3062                       t="mkdir", name="newdir")
3063         d.addBoth(self.shouldRedirect, "/THERE")
3064         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3065         d.addCallback(self.failUnlessNodeKeysAre, [])
3066         return d
3067
3068     def test_POST_bad_t(self):
3069         d = self.shouldFail2(error.Error, "POST_bad_t",
3070                              "400 Bad Request",
3071                              "POST to a directory with bad t=BOGUS",
3072                              self.POST, self.public_url + "/foo", t="BOGUS")
3073         return d
3074
3075     def test_POST_set_children(self, command_name="set_children"):
3076         contents9, n9, newuri9 = self.makefile(9)
3077         contents10, n10, newuri10 = self.makefile(10)
3078         contents11, n11, newuri11 = self.makefile(11)
3079
3080         reqbody = """{
3081                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3082                                                 "size": 0,
3083                                                 "metadata": {
3084                                                   "ctime": 1002777696.7564139,
3085                                                   "mtime": 1002777696.7564139
3086                                                  }
3087                                                } ],
3088                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3089                                                 "size": 1,
3090                                                 "metadata": {
3091                                                   "ctime": 1002777696.7564139,
3092                                                   "mtime": 1002777696.7564139
3093                                                  }
3094                                                } ],
3095                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3096                                                 "size": 2,
3097                                                 "metadata": {
3098                                                   "ctime": 1002777696.7564139,
3099                                                   "mtime": 1002777696.7564139
3100                                                  }
3101                                                } ]
3102                     }""" % (newuri9, newuri10, newuri11)
3103
3104         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3105
3106         d = client.getPage(url, method="POST", postdata=reqbody)
3107         def _then(res):
3108             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3109             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3110             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3111
3112         d.addCallback(_then)
3113         d.addErrback(self.dump_error)
3114         return d
3115
3116     def test_POST_set_children_with_hyphen(self):
3117         return self.test_POST_set_children(command_name="set-children")
3118
3119     def test_POST_link_uri(self):
3120         contents, n, newuri = self.makefile(8)
3121         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3122         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3123         d.addCallback(lambda res:
3124                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3125                                                       contents))
3126         return d
3127
3128     def test_POST_link_uri_replace(self):
3129         contents, n, newuri = self.makefile(8)
3130         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3131         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3132         d.addCallback(lambda res:
3133                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3134                                                       contents))
3135         return d
3136
3137     def test_POST_link_uri_unknown_bad(self):
3138         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3139         d.addBoth(self.shouldFail, error.Error,
3140                   "POST_link_uri_unknown_bad",
3141                   "400 Bad Request",
3142                   "unknown cap in a write slot")
3143         return d
3144
3145     def test_POST_link_uri_unknown_ro_good(self):
3146         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3147         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3148         return d
3149
3150     def test_POST_link_uri_unknown_imm_good(self):
3151         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3152         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3153         return d
3154
3155     def test_POST_link_uri_no_replace_queryarg(self):
3156         contents, n, newuri = self.makefile(8)
3157         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3158                       name="bar.txt", uri=newuri)
3159         d.addBoth(self.shouldFail, error.Error,
3160                   "POST_link_uri_no_replace_queryarg",
3161                   "409 Conflict",
3162                   "There was already a child by that name, and you asked me "
3163                   "to not replace it")
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         return d
3167
3168     def test_POST_link_uri_no_replace_field(self):
3169         contents, n, newuri = self.makefile(8)
3170         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3171                       name="bar.txt", uri=newuri)
3172         d.addBoth(self.shouldFail, error.Error,
3173                   "POST_link_uri_no_replace_field",
3174                   "409 Conflict",
3175                   "There was already a child by that name, and you asked me "
3176                   "to not replace it")
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3178         d.addCallback(self.failUnlessIsBarDotTxt)
3179         return d
3180
3181     def test_POST_delete(self, command_name='delete'):
3182         d = self._foo_node.list()
3183         def _check_before(children):
3184             self.failUnlessIn(u"bar.txt", children)
3185         d.addCallback(_check_before)
3186         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3187         d.addCallback(lambda res: self._foo_node.list())
3188         def _check_after(children):
3189             self.failIfIn(u"bar.txt", children)
3190         d.addCallback(_check_after)
3191         return d
3192
3193     def test_POST_unlink(self):
3194         return self.test_POST_delete(command_name='unlink')
3195
3196     def test_POST_rename_file(self):
3197         d = self.POST(self.public_url + "/foo", t="rename",
3198                       from_name="bar.txt", to_name='wibble.txt')
3199         d.addCallback(lambda res:
3200                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3201         d.addCallback(lambda res:
3202                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3203         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3204         d.addCallback(self.failUnlessIsBarDotTxt)
3205         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3206         d.addCallback(self.failUnlessIsBarJSON)
3207         return d
3208
3209     def test_POST_rename_file_redundant(self):
3210         d = self.POST(self.public_url + "/foo", t="rename",
3211                       from_name="bar.txt", to_name='bar.txt')
3212         d.addCallback(lambda res:
3213                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3214         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3215         d.addCallback(self.failUnlessIsBarDotTxt)
3216         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3217         d.addCallback(self.failUnlessIsBarJSON)
3218         return d
3219
3220     def test_POST_rename_file_replace(self):
3221         # rename a file and replace a directory with it
3222         d = self.POST(self.public_url + "/foo", t="rename",
3223                       from_name="bar.txt", to_name='empty')
3224         d.addCallback(lambda res:
3225                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3226         d.addCallback(lambda res:
3227                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3228         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3229         d.addCallback(self.failUnlessIsBarDotTxt)
3230         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3231         d.addCallback(self.failUnlessIsBarJSON)
3232         return d
3233
3234     def test_POST_rename_file_no_replace_queryarg(self):
3235         # rename a file and replace a directory with it
3236         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3237                       from_name="bar.txt", to_name='empty')
3238         d.addBoth(self.shouldFail, error.Error,
3239                   "POST_rename_file_no_replace_queryarg",
3240                   "409 Conflict",
3241                   "There was already a child by that name, and you asked me "
3242                   "to not replace it")
3243         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3244         d.addCallback(self.failUnlessIsEmptyJSON)
3245         return d
3246
3247     def test_POST_rename_file_no_replace_field(self):
3248         # rename a file and replace a directory with it
3249         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3250                       from_name="bar.txt", to_name='empty')
3251         d.addBoth(self.shouldFail, error.Error,
3252                   "POST_rename_file_no_replace_field",
3253                   "409 Conflict",
3254                   "There was already a child by that name, and you asked me "
3255                   "to not replace it")
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3257         d.addCallback(self.failUnlessIsEmptyJSON)
3258         return d
3259
3260     def failUnlessIsEmptyJSON(self, res):
3261         data = simplejson.loads(res)
3262         self.failUnlessEqual(data[0], "dirnode", data)
3263         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3264
3265     def test_POST_rename_file_slash_fail(self):
3266         d = self.POST(self.public_url + "/foo", t="rename",
3267                       from_name="bar.txt", to_name='kirk/spock.txt')
3268         d.addBoth(self.shouldFail, error.Error,
3269                   "test_POST_rename_file_slash_fail",
3270                   "400 Bad Request",
3271                   "to_name= may not contain a slash",
3272                   )
3273         d.addCallback(lambda res:
3274                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3275         return d
3276
3277     def test_POST_rename_dir(self):
3278         d = self.POST(self.public_url, t="rename",
3279                       from_name="foo", to_name='plunk')
3280         d.addCallback(lambda res:
3281                       self.failIfNodeHasChild(self.public_root, u"foo"))
3282         d.addCallback(lambda res:
3283                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3284         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3285         d.addCallback(self.failUnlessIsFooJSON)
3286         return d
3287
3288     def test_POST_move_file(self):
3289         d = self.POST(self.public_url + "/foo", t="move",
3290                       from_name="bar.txt", to_dir="sub")
3291         d.addCallback(lambda res:
3292                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3293         d.addCallback(lambda res:
3294                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3296         d.addCallback(self.failUnlessIsBarDotTxt)
3297         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3298         d.addCallback(self.failUnlessIsBarJSON)
3299         return d
3300
3301     def test_POST_move_file_new_name(self):
3302         d = self.POST(self.public_url + "/foo", t="move",
3303                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3304         d.addCallback(lambda res:
3305                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3306         d.addCallback(lambda res:
3307                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3308         d.addCallback(lambda res:
3309                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3310         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3311         d.addCallback(self.failUnlessIsBarDotTxt)
3312         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3313         d.addCallback(self.failUnlessIsBarJSON)
3314         return d
3315
3316     def test_POST_move_file_replace(self):
3317         d = self.POST(self.public_url + "/foo", t="move",
3318                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3319         d.addCallback(lambda res:
3320                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3321         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3322         d.addCallback(self.failUnlessIsBarDotTxt)
3323         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3324         d.addCallback(self.failUnlessIsBarJSON)
3325         return d
3326
3327     def test_POST_move_file_no_replace(self):
3328         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3329                              "409 Conflict",
3330                              "There was already a child by that name, and you asked me to not replace it",
3331                              self.POST, self.public_url + "/foo", t="move",
3332                              replace="false", from_name="bar.txt",
3333                              to_name="baz.txt", to_dir="sub")
3334         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3335         d.addCallback(self.failUnlessIsBarDotTxt)
3336         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3337         d.addCallback(self.failUnlessIsBarJSON)
3338         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3339         d.addCallback(self.failUnlessIsSubBazDotTxt)
3340         return d
3341
3342     def test_POST_move_file_slash_fail(self):
3343         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3344                              "400 Bad Request",
3345                              "to_name= may not contain a slash",
3346                              self.POST, self.public_url + "/foo", t="move",
3347                              from_name="bar.txt",
3348                              to_name="slash/fail.txt", to_dir="sub")
3349         d.addCallback(lambda res:
3350                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3351         d.addCallback(lambda res:
3352                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3353         d.addCallback(lambda ign:
3354                       self.shouldFail2(error.Error,
3355                                        "test_POST_rename_file_slash_fail2",
3356                                        "400 Bad Request",
3357                                        "from_name= may not contain a slash",
3358                                        self.POST, self.public_url + "/foo",
3359                                        t="move",
3360                                        from_name="nope/bar.txt",
3361                                        to_name="fail.txt", to_dir="sub"))
3362         return d
3363
3364     def test_POST_move_file_no_target(self):
3365         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3366                              "400 Bad Request",
3367                              "move requires from_name and to_dir",
3368                              self.POST, self.public_url + "/foo", t="move",
3369                              from_name="bar.txt", to_name="baz.txt")
3370         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3371         d.addCallback(self.failUnlessIsBarDotTxt)
3372         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3373         d.addCallback(self.failUnlessIsBarJSON)
3374         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3375         d.addCallback(self.failUnlessIsBazDotTxt)
3376         return d
3377
3378     def test_POST_move_file_bad_target_type(self):
3379         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3380                              "400 Bad Request", "invalid target_type parameter",
3381                              self.POST,
3382                              self.public_url + "/foo", t="move",
3383                              target_type="*D", from_name="bar.txt",
3384                              to_dir="sub")
3385         return d
3386
3387     def test_POST_move_file_multi_level(self):
3388         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3389         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3390                       from_name="bar.txt", to_dir="sub/level2"))
3391         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3392         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3393         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3394         d.addCallback(self.failUnlessIsBarDotTxt)
3395         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3396         d.addCallback(self.failUnlessIsBarJSON)
3397         return d
3398
3399     def test_POST_move_file_to_uri(self):
3400         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3401                       from_name="bar.txt", to_dir=self._sub_uri)
3402         d.addCallback(lambda res:
3403                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3404         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3405         d.addCallback(self.failUnlessIsBarDotTxt)
3406         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3407         d.addCallback(self.failUnlessIsBarJSON)
3408         return d
3409
3410     def test_POST_move_file_to_nonexist_dir(self):
3411         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3412                             "404 Not Found", "No such child: nopechucktesta",
3413                             self.POST, self.public_url + "/foo", t="move",
3414                             from_name="bar.txt", to_dir="nopechucktesta")
3415         return d
3416
3417     def test_POST_move_file_into_file(self):
3418         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3419                              "400 Bad Request", "to_dir is not a directory",
3420                              self.POST, self.public_url + "/foo", t="move",
3421                              from_name="bar.txt", to_dir="baz.txt")
3422         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3423         d.addCallback(self.failUnlessIsBazDotTxt)
3424         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3425         d.addCallback(self.failUnlessIsBarDotTxt)
3426         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3427         d.addCallback(self.failUnlessIsBarJSON)
3428         return d
3429
3430     def test_POST_move_file_to_bad_uri(self):
3431         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3432                               "400 Bad Request", "to_dir is not a directory",
3433                               self.POST, self.public_url + "/foo", t="move",
3434                               from_name="bar.txt", target_type="uri",
3435                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3436         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3437         d.addCallback(self.failUnlessIsBarDotTxt)
3438         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3439         d.addCallback(self.failUnlessIsBarJSON)
3440         return d
3441
3442     def test_POST_move_dir(self):
3443         d = self.POST(self.public_url + "/foo", t="move",
3444                       from_name="bar.txt", to_dir="empty")
3445         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3446                       t="move", from_name="empty", to_dir="sub"))
3447         d.addCallback(lambda res:
3448                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3449         d.addCallback(lambda res:
3450                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3451         d.addCallback(lambda res:
3452                       self._sub_node.get_child_at_path(u"empty"))
3453         d.addCallback(lambda node:
3454                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3455         d.addCallback(lambda res:
3456                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3457         d.addCallback(self.failUnlessIsBarDotTxt)
3458         return d
3459
3460     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3461         """ If target is not None then the redirection has to go to target.  If
3462         statuscode is not None then the redirection has to be accomplished with
3463         that HTTP status code."""
3464         if not isinstance(res, failure.Failure):
3465             to_where = (target is None) and "somewhere" or ("to " + target)
3466             self.fail("%s: we were expecting to get redirected %s, not get an"
3467                       " actual page: %s" % (which, to_where, res))
3468         res.trap(error.PageRedirect)
3469         if statuscode is not None:
3470             self.failUnlessReallyEqual(res.value.status, statuscode,
3471                                        "%s: not a redirect" % which)
3472         if target is not None:
3473             # the PageRedirect does not seem to capture the uri= query arg
3474             # properly, so we can't check for it.
3475             realtarget = self.webish_url + target
3476             self.failUnlessReallyEqual(res.value.location, realtarget,
3477                                        "%s: wrong target" % which)
3478         return res.value.location
3479
3480     def test_GET_URI_form(self):
3481         base = "/uri?uri=%s" % self._bar_txt_uri
3482         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3483         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3484         d = self.GET(base)
3485         d.addBoth(self.shouldRedirect, targetbase)
3486         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3487         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3488         d.addCallback(lambda res: self.GET(base+"&t=json"))
3489         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3490         d.addCallback(self.log, "about to get file by uri")
3491         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3492         d.addCallback(self.failUnlessIsBarDotTxt)
3493         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3494         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3495                                            followRedirect=True))
3496         d.addCallback(self.failUnlessIsFooJSON)
3497         d.addCallback(self.log, "got dir by uri")
3498
3499         return d
3500
3501     def test_GET_URI_form_bad(self):
3502         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3503                              "400 Bad Request", "GET /uri requires uri=",
3504                              self.GET, "/uri")
3505         return d
3506
3507     def test_GET_rename_form(self):
3508         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3509                      followRedirect=True)
3510         def _check(res):
3511             self.failUnlessIn('name="when_done" value="."', res)
3512             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3513             self.failUnlessIn(FAVICON_MARKUP, res)
3514         d.addCallback(_check)
3515         return d
3516
3517     def log(self, res, msg):
3518         #print "MSG: %s  RES: %s" % (msg, res)
3519         log.msg(msg)
3520         return res
3521
3522     def test_GET_URI_URL(self):
3523         base = "/uri/%s" % self._bar_txt_uri
3524         d = self.GET(base)
3525         d.addCallback(self.failUnlessIsBarDotTxt)
3526         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3527         d.addCallback(self.failUnlessIsBarDotTxt)
3528         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3529         d.addCallback(self.failUnlessIsBarDotTxt)
3530         return d
3531
3532     def test_GET_URI_URL_dir(self):
3533         base = "/uri/%s?t=json" % self._foo_uri
3534         d = self.GET(base)
3535         d.addCallback(self.failUnlessIsFooJSON)
3536         return d
3537
3538     def test_GET_URI_URL_missing(self):
3539         base = "/uri/%s" % self._bad_file_uri
3540         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3541                                  http.GONE, None, "NotEnoughSharesError",
3542                                  self.GET, base)
3543         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3544         # here? we must arrange for a download to fail after target.open()
3545         # has been called, and then inspect the response to see that it is
3546         # shorter than we expected.
3547         return d
3548
3549     def test_PUT_DIRURL_uri(self):
3550         d = self.s.create_dirnode()
3551         def _made_dir(dn):
3552             new_uri = dn.get_uri()
3553             # replace /foo with a new (empty) directory
3554             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3555             d.addCallback(lambda res:
3556                           self.failUnlessReallyEqual(res.strip(), new_uri))
3557             d.addCallback(lambda res:
3558                           self.failUnlessRWChildURIIs(self.public_root,
3559                                                       u"foo",
3560                                                       new_uri))
3561             return d
3562         d.addCallback(_made_dir)
3563         return d
3564
3565     def test_PUT_DIRURL_uri_noreplace(self):
3566         d = self.s.create_dirnode()
3567         def _made_dir(dn):
3568             new_uri = dn.get_uri()
3569             # replace /foo with a new (empty) directory, but ask that
3570             # replace=false, so it should fail
3571             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3572                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3573                                  self.PUT,
3574                                  self.public_url + "/foo?t=uri&replace=false",
3575                                  new_uri)
3576             d.addCallback(lambda res:
3577                           self.failUnlessRWChildURIIs(self.public_root,
3578                                                       u"foo",
3579                                                       self._foo_uri))
3580             return d
3581         d.addCallback(_made_dir)
3582         return d
3583
3584     def test_PUT_DIRURL_bad_t(self):
3585         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3586                              "400 Bad Request", "PUT to a directory",
3587                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3588         d.addCallback(lambda res:
3589                       self.failUnlessRWChildURIIs(self.public_root,
3590                                                   u"foo",
3591                                                   self._foo_uri))
3592         return d
3593
3594     def test_PUT_NEWFILEURL_uri(self):
3595         contents, n, new_uri = self.makefile(8)
3596         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3597         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3598         d.addCallback(lambda res:
3599                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3600                                                       contents))
3601         return d
3602
3603     def test_PUT_NEWFILEURL_mdmf(self):
3604         new_contents = self.NEWFILE_CONTENTS * 300000
3605         d = self.PUT(self.public_url + \
3606                      "/foo/mdmf.txt?format=mdmf",
3607                      new_contents)
3608         d.addCallback(lambda ignored:
3609             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3610         def _got_json(json):
3611             data = simplejson.loads(json)
3612             data = data[1]
3613             self.failUnlessIn("format", data)
3614             self.failUnlessEqual(data["format"], "MDMF")
3615             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3616             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3617         d.addCallback(_got_json)
3618         return d
3619
3620     def test_PUT_NEWFILEURL_sdmf(self):
3621         new_contents = self.NEWFILE_CONTENTS * 300000
3622         d = self.PUT(self.public_url + \
3623                      "/foo/sdmf.txt?format=sdmf",
3624                      new_contents)
3625         d.addCallback(lambda ignored:
3626             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3627         def _got_json(json):
3628             data = simplejson.loads(json)
3629             data = data[1]
3630             self.failUnlessIn("format", data)
3631             self.failUnlessEqual(data["format"], "SDMF")
3632         d.addCallback(_got_json)
3633         return d
3634
3635     def test_PUT_NEWFILEURL_bad_format(self):
3636        new_contents = self.NEWFILE_CONTENTS * 300000
3637        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3638                                    400, "Bad Request", "Unknown format: foo",
3639                                    self.PUT, self.public_url + \
3640                                    "/foo/foo.txt?format=foo",
3641                                    new_contents)
3642
3643     def test_PUT_NEWFILEURL_uri_replace(self):
3644         contents, n, new_uri = self.makefile(8)
3645         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3646         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3647         d.addCallback(lambda res:
3648                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3649                                                       contents))
3650         return d
3651
3652     def test_PUT_NEWFILEURL_uri_no_replace(self):
3653         contents, n, new_uri = self.makefile(8)
3654         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3655         d.addBoth(self.shouldFail, error.Error,
3656                   "PUT_NEWFILEURL_uri_no_replace",
3657                   "409 Conflict",
3658                   "There was already a child by that name, and you asked me "
3659                   "to not replace it")
3660         return d
3661
3662     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3663         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3664         d.addBoth(self.shouldFail, error.Error,
3665                   "POST_put_uri_unknown_bad",
3666                   "400 Bad Request",
3667                   "unknown cap in a write slot")
3668         return d
3669
3670     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3671         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3672         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3673                       u"put-future-ro.txt")
3674         return d
3675
3676     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3677         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3678         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3679                       u"put-future-imm.txt")
3680         return d
3681
3682     def test_PUT_NEWFILE_URI(self):
3683         file_contents = "New file contents here\n"
3684         d = self.PUT("/uri", file_contents)
3685         def _check(uri):
3686             assert isinstance(uri, str), uri
3687             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3688             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3689                                        file_contents)
3690             return self.GET("/uri/%s" % uri)
3691         d.addCallback(_check)
3692         def _check2(res):
3693             self.failUnlessReallyEqual(res, file_contents)
3694         d.addCallback(_check2)
3695         return d
3696
3697     def test_PUT_NEWFILE_URI_not_mutable(self):
3698         file_contents = "New file contents here\n"
3699         d = self.PUT("/uri?mutable=false", file_contents)
3700         def _check(uri):
3701             assert isinstance(uri, str), uri
3702             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3703             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3704                                        file_contents)
3705             return self.GET("/uri/%s" % uri)
3706         d.addCallback(_check)
3707         def _check2(res):
3708             self.failUnlessReallyEqual(res, file_contents)
3709         d.addCallback(_check2)
3710         return d
3711
3712     def test_PUT_NEWFILE_URI_only_PUT(self):
3713         d = self.PUT("/uri?t=bogus", "")
3714         d.addBoth(self.shouldFail, error.Error,
3715                   "PUT_NEWFILE_URI_only_PUT",
3716                   "400 Bad Request",
3717                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3718         return d
3719
3720     def test_PUT_NEWFILE_URI_mutable(self):
3721         file_contents = "New file contents here\n"
3722         d = self.PUT("/uri?mutable=true", file_contents)
3723         def _check1(filecap):
3724             filecap = filecap.strip()
3725             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3726             self.filecap = filecap
3727             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3728             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3729             n = self.s.create_node_from_uri(filecap)
3730             return n.download_best_version()
3731         d.addCallback(_check1)
3732         def _check2(data):
3733             self.failUnlessReallyEqual(data, file_contents)
3734             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3735         d.addCallback(_check2)
3736         def _check3(res):
3737             self.failUnlessReallyEqual(res, file_contents)
3738         d.addCallback(_check3)
3739         return d
3740
3741     def test_PUT_mkdir(self):
3742         d = self.PUT("/uri?t=mkdir", "")
3743         def _check(uri):
3744             n = self.s.create_node_from_uri(uri.strip())
3745             d2 = self.failUnlessNodeKeysAre(n, [])
3746             d2.addCallback(lambda res:
3747                            self.GET("/uri/%s?t=json" % uri))
3748             return d2
3749         d.addCallback(_check)
3750         d.addCallback(self.failUnlessIsEmptyJSON)
3751         return d
3752
3753     def test_PUT_mkdir_mdmf(self):
3754         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3755         def _got(res):
3756             u = uri.from_string(res)
3757             # Check that this is an MDMF writecap
3758             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3759         d.addCallback(_got)
3760         return d
3761
3762     def test_PUT_mkdir_sdmf(self):
3763         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3764         def _got(res):
3765             u = uri.from_string(res)
3766             self.failUnlessIsInstance(u, uri.DirectoryURI)
3767         d.addCallback(_got)
3768         return d
3769
3770     def test_PUT_mkdir_bad_format(self):
3771         return self.shouldHTTPError("PUT_mkdir_bad_format",
3772                                     400, "Bad Request", "Unknown format: foo",
3773                                     self.PUT, "/uri?t=mkdir&format=foo",
3774                                     "")
3775
3776     def test_POST_check(self):
3777         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3778         def _done(res):
3779             # this returns a string form of the results, which are probably
3780             # None since we're using fake filenodes.
3781             # TODO: verify that the check actually happened, by changing
3782             # FakeCHKFileNode to count how many times .check() has been
3783             # called.
3784             pass
3785         d.addCallback(_done)
3786         return d
3787
3788
3789     def test_PUT_update_at_offset(self):
3790         file_contents = "test file" * 100000 # about 900 KiB
3791         d = self.PUT("/uri?mutable=true", file_contents)
3792         def _then(filecap):
3793             self.filecap = filecap
3794             new_data = file_contents[:100]
3795             new = "replaced and so on"
3796             new_data += new
3797             new_data += file_contents[len(new_data):]
3798             assert len(new_data) == len(file_contents)
3799             self.new_data = new_data
3800         d.addCallback(_then)
3801         d.addCallback(lambda ignored:
3802             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3803                      "replaced and so on"))
3804         def _get_data(filecap):
3805             n = self.s.create_node_from_uri(filecap)
3806             return n.download_best_version()
3807         d.addCallback(_get_data)
3808         d.addCallback(lambda results:
3809             self.failUnlessEqual(results, self.new_data))
3810         # Now try appending things to the file
3811         d.addCallback(lambda ignored:
3812             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3813                      "puppies" * 100))
3814         d.addCallback(_get_data)
3815         d.addCallback(lambda results:
3816             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3817         # and try replacing the beginning of the file
3818         d.addCallback(lambda ignored:
3819             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3820         d.addCallback(_get_data)
3821         d.addCallback(lambda results:
3822             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3823         return d
3824
3825     def test_PUT_update_at_invalid_offset(self):
3826         file_contents = "test file" * 100000 # about 900 KiB
3827         d = self.PUT("/uri?mutable=true", file_contents)
3828         def _then(filecap):
3829             self.filecap = filecap
3830         d.addCallback(_then)
3831         # Negative offsets should cause an error.
3832         d.addCallback(lambda ignored:
3833             self.shouldHTTPError("PUT_update_at_invalid_offset",
3834                                  400, "Bad Request",
3835                                  "Invalid offset",
3836                                  self.PUT,
3837                                  "/uri/%s?offset=-1" % self.filecap,
3838                                  "foo"))
3839         return d
3840
3841     def test_PUT_update_at_offset_immutable(self):
3842         file_contents = "Test file" * 100000
3843         d = self.PUT("/uri", file_contents)
3844         def _then(filecap):
3845             self.filecap = filecap
3846         d.addCallback(_then)
3847         d.addCallback(lambda ignored:
3848             self.shouldHTTPError("PUT_update_at_offset_immutable",
3849                                  400, "Bad Request",
3850                                  "immutable",
3851                                  self.PUT,
3852                                  "/uri/%s?offset=50" % self.filecap,
3853                                  "foo"))
3854         return d
3855
3856
3857     def test_bad_method(self):
3858         url = self.webish_url + self.public_url + "/foo/bar.txt"
3859         d = self.shouldHTTPError("bad_method",
3860                                  501, "Not Implemented",
3861                                  "I don't know how to treat a BOGUS request.",
3862                                  client.getPage, url, method="BOGUS")
3863         return d
3864
3865     def test_short_url(self):
3866         url = self.webish_url + "/uri"
3867         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3868                                  "I don't know how to treat a DELETE request.",
3869                                  client.getPage, url, method="DELETE")
3870         return d
3871
3872     def test_ophandle_bad(self):
3873         url = self.webish_url + "/operations/bogus?t=status"
3874         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3875                                  "unknown/expired handle 'bogus'",
3876                                  client.getPage, url)
3877         return d
3878
3879     def test_ophandle_cancel(self):
3880         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3881                       followRedirect=True)
3882         d.addCallback(lambda ignored:
3883                       self.GET("/operations/128?t=status&output=JSON"))
3884         def _check1(res):
3885             data = simplejson.loads(res)
3886             self.failUnless("finished" in data, res)
3887             monitor = self.ws.root.child_operations.handles["128"][0]
3888             d = self.POST("/operations/128?t=cancel&output=JSON")
3889             def _check2(res):
3890                 data = simplejson.loads(res)
3891                 self.failUnless("finished" in data, res)
3892                 # t=cancel causes the handle to be forgotten
3893                 self.failUnless(monitor.is_cancelled())
3894             d.addCallback(_check2)
3895             return d
3896         d.addCallback(_check1)
3897         d.addCallback(lambda ignored:
3898                       self.shouldHTTPError("ophandle_cancel",
3899                                            404, "404 Not Found",
3900                                            "unknown/expired handle '128'",
3901                                            self.GET,
3902                                            "/operations/128?t=status&output=JSON"))
3903         return d
3904
3905     def test_ophandle_retainfor(self):
3906         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3907                       followRedirect=True)
3908         d.addCallback(lambda ignored:
3909                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3910         def _check1(res):
3911             data = simplejson.loads(res)
3912             self.failUnless("finished" in data, res)
3913         d.addCallback(_check1)
3914         # the retain-for=0 will cause the handle to be expired very soon
3915         d.addCallback(lambda ign:
3916             self.clock.advance(2.0))
3917         d.addCallback(lambda ignored:
3918                       self.shouldHTTPError("ophandle_retainfor",
3919                                            404, "404 Not Found",
3920                                            "unknown/expired handle '129'",
3921                                            self.GET,
3922                                            "/operations/129?t=status&output=JSON"))
3923         return d
3924
3925     def test_ophandle_release_after_complete(self):
3926         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3927                       followRedirect=True)
3928         d.addCallback(self.wait_for_operation, "130")
3929         d.addCallback(lambda ignored:
3930                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3931         # the release-after-complete=true will cause the handle to be expired
3932         d.addCallback(lambda ignored:
3933                       self.shouldHTTPError("ophandle_release_after_complete",
3934                                            404, "404 Not Found",
3935                                            "unknown/expired handle '130'",
3936                                            self.GET,
3937                                            "/operations/130?t=status&output=JSON"))
3938         return d
3939
3940     def test_uncollected_ophandle_expiration(self):
3941         # uncollected ophandles should expire after 4 days
3942         def _make_uncollected_ophandle(ophandle):
3943             d = self.POST(self.public_url +
3944                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3945                           followRedirect=False)
3946             # When we start the operation, the webapi server will want
3947             # to redirect us to the page for the ophandle, so we get
3948             # confirmation that the operation has started. If the
3949             # manifest operation has finished by the time we get there,
3950             # following that redirect (by setting followRedirect=True
3951             # above) has the side effect of collecting the ophandle that
3952             # we've just created, which means that we can't use the
3953             # ophandle to test the uncollected timeout anymore. So,
3954             # instead, catch the 302 here and don't follow it.
3955             d.addBoth(self.should302, "uncollected_ophandle_creation")
3956             return d
3957         # Create an ophandle, don't collect it, then advance the clock by
3958         # 4 days - 1 second and make sure that the ophandle is still there.
3959         d = _make_uncollected_ophandle(131)
3960         d.addCallback(lambda ign:
3961             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3962         d.addCallback(lambda ign:
3963             self.GET("/operations/131?t=status&output=JSON"))
3964         def _check1(res):
3965             data = simplejson.loads(res)
3966             self.failUnless("finished" in data, res)
3967         d.addCallback(_check1)
3968         # Create an ophandle, don't collect it, then try to collect it
3969         # after 4 days. It should be gone.
3970         d.addCallback(lambda ign:
3971             _make_uncollected_ophandle(132))
3972         d.addCallback(lambda ign:
3973             self.clock.advance(96*60*60))
3974         d.addCallback(lambda ign:
3975             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3976                                  404, "404 Not Found",
3977                                  "unknown/expired handle '132'",
3978                                  self.GET,
3979                                  "/operations/132?t=status&output=JSON"))
3980         return d
3981
3982     def test_collected_ophandle_expiration(self):
3983         # collected ophandles should expire after 1 day
3984         def _make_collected_ophandle(ophandle):
3985             d = self.POST(self.public_url +
3986                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3987                           followRedirect=True)
3988             # By following the initial redirect, we collect the ophandle
3989             # we've just created.
3990             return d
3991         # Create a collected ophandle, then collect it after 23 hours
3992         # and 59 seconds to make sure that it is still there.
3993         d = _make_collected_ophandle(133)
3994         d.addCallback(lambda ign:
3995             self.clock.advance((24*60*60) - 1))
3996         d.addCallback(lambda ign:
3997             self.GET("/operations/133?t=status&output=JSON"))
3998         def _check1(res):
3999             data = simplejson.loads(res)
4000             self.failUnless("finished" in data, res)
4001         d.addCallback(_check1)
4002         # Create another uncollected ophandle, then try to collect it
4003         # after 24 hours to make sure that it is gone.
4004         d.addCallback(lambda ign:
4005             _make_collected_ophandle(134))
4006         d.addCallback(lambda ign:
4007             self.clock.advance(24*60*60))
4008         d.addCallback(lambda ign:
4009             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4010                                  404, "404 Not Found",
4011                                  "unknown/expired handle '134'",
4012                                  self.GET,
4013                                  "/operations/134?t=status&output=JSON"))
4014         return d
4015
4016     def test_incident(self):
4017         d = self.POST("/report_incident", details="eek")
4018         def _done(res):
4019             self.failIfIn("<html>", res)
4020             self.failUnlessIn("Thank you for your report!", res)
4021         d.addCallback(_done)
4022         return d
4023
4024     def test_static(self):
4025         webdir = os.path.join(self.staticdir, "subdir")
4026         fileutil.make_dirs(webdir)
4027         f = open(os.path.join(webdir, "hello.txt"), "wb")
4028         f.write("hello")
4029         f.close()
4030
4031         d = self.GET("/static/subdir/hello.txt")
4032         def _check(res):
4033             self.failUnlessReallyEqual(res, "hello")
4034         d.addCallback(_check)
4035         return d
4036
4037
4038 class IntroducerWeb(unittest.TestCase):
4039     def setUp(self):
4040         self.node = None
4041
4042     def tearDown(self):
4043         d = defer.succeed(None)
4044         if self.node:
4045             d.addCallback(lambda ign: self.node.stopService())
4046         d.addCallback(flushEventualQueue)
4047         return d
4048
4049     def test_welcome(self):
4050         basedir = "web.IntroducerWeb.test_welcome"
4051         os.mkdir(basedir)
4052         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4053         self.node = IntroducerNode(basedir)
4054         self.ws = self.node.getServiceNamed("webish")
4055
4056         d = fireEventually(None)
4057         d.addCallback(lambda ign: self.node.startService())
4058         d.addCallback(lambda ign: self.node.when_tub_ready())
4059
4060         d.addCallback(lambda ign: self.GET("/"))
4061         def _check(res):
4062             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4063             self.failUnlessIn(FAVICON_MARKUP, res)
4064         d.addCallback(_check)
4065         return d
4066
4067     def GET(self, urlpath, followRedirect=False, return_response=False,
4068             **kwargs):
4069         # if return_response=True, this fires with (data, statuscode,
4070         # respheaders) instead of just data.
4071         assert not isinstance(urlpath, unicode)
4072         url = self.ws.getURL().rstrip('/') + urlpath
4073         factory = HTTPClientGETFactory(url, method="GET",
4074                                        followRedirect=followRedirect, **kwargs)
4075         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4076         d = factory.deferred
4077         def _got_data(data):
4078             return (data, factory.status, factory.response_headers)
4079         if return_response:
4080             d.addCallback(_got_data)
4081         return factory.deferred
4082
4083
4084 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4085     def test_load_file(self):
4086         # This will raise an exception unless a well-formed XML file is found under that name.
4087         common.getxmlfile('directory.xhtml').load()
4088
4089     def test_parse_replace_arg(self):
4090         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4091         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4092         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4093                                    "only-files")
4094         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4095                         common.parse_replace_arg, "only_fles")
4096
4097     def test_abbreviate_time(self):
4098         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4099         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4100         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4101         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4102         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4103         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4104
4105     def test_compute_rate(self):
4106         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4107         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4108         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4109         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4110         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4111         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4112         self.shouldFail(AssertionError, "test_compute_rate", "",
4113                         common.compute_rate, -100, 10)
4114         self.shouldFail(AssertionError, "test_compute_rate", "",
4115                         common.compute_rate, 100, -10)
4116
4117         # Sanity check
4118         rate = common.compute_rate(10*1000*1000, 1)
4119         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4120
4121     def test_abbreviate_rate(self):
4122         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4123         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4124         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4125         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4126
4127     def test_abbreviate_size(self):
4128         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4129         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4130         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4131         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4132         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4133
4134     def test_plural(self):
4135         def convert(s):
4136             return "%d second%s" % (s, status.plural(s))
4137         self.failUnlessReallyEqual(convert(0), "0 seconds")
4138         self.failUnlessReallyEqual(convert(1), "1 second")
4139         self.failUnlessReallyEqual(convert(2), "2 seconds")
4140         def convert2(s):
4141             return "has share%s: %s" % (status.plural(s), ",".join(s))
4142         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4143         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4144         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4145
4146
4147 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4148
4149     def CHECK(self, ign, which, args, clientnum=0):
4150         fileurl = self.fileurls[which]
4151         url = fileurl + "?" + args
4152         return self.GET(url, method="POST", clientnum=clientnum)
4153
4154     def test_filecheck(self):
4155         self.basedir = "web/Grid/filecheck"
4156         self.set_up_grid()
4157         c0 = self.g.clients[0]
4158         self.uris = {}
4159         DATA = "data" * 100
4160         d = c0.upload(upload.Data(DATA, convergence=""))
4161         def _stash_uri(ur, which):
4162             self.uris[which] = ur.uri
4163         d.addCallback(_stash_uri, "good")
4164         d.addCallback(lambda ign:
4165                       c0.upload(upload.Data(DATA+"1", convergence="")))
4166         d.addCallback(_stash_uri, "sick")
4167         d.addCallback(lambda ign:
4168                       c0.upload(upload.Data(DATA+"2", convergence="")))
4169         d.addCallback(_stash_uri, "dead")
4170         def _stash_mutable_uri(n, which):
4171             self.uris[which] = n.get_uri()
4172             assert isinstance(self.uris[which], str)
4173         d.addCallback(lambda ign:
4174             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4175         d.addCallback(_stash_mutable_uri, "corrupt")
4176         d.addCallback(lambda ign:
4177                       c0.upload(upload.Data("literal", convergence="")))
4178         d.addCallback(_stash_uri, "small")
4179         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4180         d.addCallback(_stash_mutable_uri, "smalldir")
4181
4182         def _compute_fileurls(ignored):
4183             self.fileurls = {}
4184             for which in self.uris:
4185                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4186         d.addCallback(_compute_fileurls)
4187
4188         def _clobber_shares(ignored):
4189             good_shares = self.find_uri_shares(self.uris["good"])
4190             self.failUnlessReallyEqual(len(good_shares), 10)
4191             sick_shares = self.find_uri_shares(self.uris["sick"])
4192             os.unlink(sick_shares[0][2])
4193             dead_shares = self.find_uri_shares(self.uris["dead"])
4194             for i in range(1, 10):
4195                 os.unlink(dead_shares[i][2])
4196             c_shares = self.find_uri_shares(self.uris["corrupt"])
4197             cso = CorruptShareOptions()
4198             cso.stdout = StringIO()
4199             cso.parseOptions([c_shares[0][2]])
4200             corrupt_share(cso)
4201         d.addCallback(_clobber_shares)
4202
4203         d.addCallback(self.CHECK, "good", "t=check")
4204         def _got_html_good(res):
4205             self.failUnlessIn("Healthy", res)
4206             self.failIfIn("Not Healthy", res)
4207             self.failUnlessIn(FAVICON_MARKUP, res)
4208         d.addCallback(_got_html_good)
4209         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4210         def _got_html_good_return_to(res):
4211             self.failUnlessIn("Healthy", res)
4212             self.failIfIn("Not Healthy", res)
4213             self.failUnlessIn('<a href="somewhere">Return to file', res)
4214         d.addCallback(_got_html_good_return_to)
4215         d.addCallback(self.CHECK, "good", "t=check&output=json")
4216         def _got_json_good(res):
4217             r = simplejson.loads(res)
4218             self.failUnlessEqual(r["summary"], "Healthy")
4219             self.failUnless(r["results"]["healthy"])
4220             self.failIf(r["results"]["needs-rebalancing"])
4221             self.failUnless(r["results"]["recoverable"])
4222         d.addCallback(_got_json_good)
4223
4224         d.addCallback(self.CHECK, "small", "t=check")
4225         def _got_html_small(res):
4226             self.failUnlessIn("Literal files are always healthy", res)
4227             self.failIfIn("Not Healthy", res)
4228         d.addCallback(_got_html_small)
4229         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4230         def _got_html_small_return_to(res):
4231             self.failUnlessIn("Literal files are always healthy", res)
4232             self.failIfIn("Not Healthy", res)
4233             self.failUnlessIn('<a href="somewhere">Return to file', res)
4234         d.addCallback(_got_html_small_return_to)
4235         d.addCallback(self.CHECK, "small", "t=check&output=json")
4236         def _got_json_small(res):
4237             r = simplejson.loads(res)
4238             self.failUnlessEqual(r["storage-index"], "")
4239             self.failUnless(r["results"]["healthy"])
4240         d.addCallback(_got_json_small)
4241
4242         d.addCallback(self.CHECK, "smalldir", "t=check")
4243         def _got_html_smalldir(res):
4244             self.failUnlessIn("Literal files are always healthy", res)
4245             self.failIfIn("Not Healthy", res)
4246         d.addCallback(_got_html_smalldir)
4247         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4248         def _got_json_smalldir(res):
4249             r = simplejson.loads(res)
4250             self.failUnlessEqual(r["storage-index"], "")
4251             self.failUnless(r["results"]["healthy"])
4252         d.addCallback(_got_json_smalldir)
4253
4254         d.addCallback(self.CHECK, "sick", "t=check")
4255         def _got_html_sick(res):
4256             self.failUnlessIn("Not Healthy", res)
4257         d.addCallback(_got_html_sick)
4258         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4259         def _got_json_sick(res):
4260             r = simplejson.loads(res)
4261             self.failUnlessEqual(r["summary"],
4262                                  "Not Healthy: 9 shares (enc 3-of-10)")
4263             self.failIf(r["results"]["healthy"])
4264             self.failIf(r["results"]["needs-rebalancing"])
4265             self.failUnless(r["results"]["recoverable"])
4266         d.addCallback(_got_json_sick)
4267
4268         d.addCallback(self.CHECK, "dead", "t=check")
4269         def _got_html_dead(res):
4270             self.failUnlessIn("Not Healthy", res)
4271         d.addCallback(_got_html_dead)
4272         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4273         def _got_json_dead(res):
4274             r = simplejson.loads(res)
4275             self.failUnlessEqual(r["summary"],
4276                                  "Not Healthy: 1 shares (enc 3-of-10)")
4277             self.failIf(r["results"]["healthy"])
4278             self.failIf(r["results"]["needs-rebalancing"])
4279             self.failIf(r["results"]["recoverable"])
4280         d.addCallback(_got_json_dead)
4281
4282         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4283         def _got_html_corrupt(res):
4284             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4285         d.addCallback(_got_html_corrupt)
4286         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4287         def _got_json_corrupt(res):
4288             r = simplejson.loads(res)
4289             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4290             self.failIf(r["results"]["healthy"])
4291             self.failUnless(r["results"]["recoverable"])
4292             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4293             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4294         d.addCallback(_got_json_corrupt)
4295
4296         d.addErrback(self.explain_web_error)
4297         return d
4298
4299     def test_repair_html(self):
4300         self.basedir = "web/Grid/repair_html"
4301         self.set_up_grid()
4302         c0 = self.g.clients[0]
4303         self.uris = {}
4304         DATA = "data" * 100
4305         d = c0.upload(upload.Data(DATA, convergence=""))
4306         def _stash_uri(ur, which):
4307             self.uris[which] = ur.uri
4308         d.addCallback(_stash_uri, "good")
4309         d.addCallback(lambda ign:
4310                       c0.upload(upload.Data(DATA+"1", convergence="")))
4311         d.addCallback(_stash_uri, "sick")
4312         d.addCallback(lambda ign:
4313                       c0.upload(upload.Data(DATA+"2", convergence="")))
4314         d.addCallback(_stash_uri, "dead")
4315         def _stash_mutable_uri(n, which):
4316             self.uris[which] = n.get_uri()
4317             assert isinstance(self.uris[which], str)
4318         d.addCallback(lambda ign:
4319             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4320         d.addCallback(_stash_mutable_uri, "corrupt")
4321
4322         def _compute_fileurls(ignored):
4323             self.fileurls = {}
4324             for which in self.uris:
4325                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4326         d.addCallback(_compute_fileurls)
4327
4328         def _clobber_shares(ignored):
4329             good_shares = self.find_uri_shares(self.uris["good"])
4330             self.failUnlessReallyEqual(len(good_shares), 10)
4331             sick_shares = self.find_uri_shares(self.uris["sick"])
4332             os.unlink(sick_shares[0][2])
4333             dead_shares = self.find_uri_shares(self.uris["dead"])
4334             for i in range(1, 10):
4335                 os.unlink(dead_shares[i][2])
4336             c_shares = self.find_uri_shares(self.uris["corrupt"])
4337             cso = CorruptShareOptions()
4338             cso.stdout = StringIO()
4339             cso.parseOptions([c_shares[0][2]])
4340             corrupt_share(cso)
4341         d.addCallback(_clobber_shares)
4342
4343         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4344         def _got_html_good(res):
4345             self.failUnlessIn("Healthy", res)
4346             self.failIfIn("Not Healthy", res)
4347             self.failUnlessIn("No repair necessary", res)
4348             self.failUnlessIn(FAVICON_MARKUP, res)
4349         d.addCallback(_got_html_good)
4350
4351         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4352         def _got_html_sick(res):
4353             self.failUnlessIn("Healthy : healthy", res)
4354             self.failIfIn("Not Healthy", res)
4355             self.failUnlessIn("Repair successful", res)
4356         d.addCallback(_got_html_sick)
4357
4358         # repair of a dead file will fail, of course, but it isn't yet
4359         # clear how this should be reported. Right now it shows up as
4360         # a "410 Gone".
4361         #
4362         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4363         #def _got_html_dead(res):
4364         #    print res
4365         #    self.failUnlessIn("Healthy : healthy", res)
4366         #    self.failIfIn("Not Healthy", res)
4367         #    self.failUnlessIn("No repair necessary", res)
4368         #d.addCallback(_got_html_dead)
4369
4370         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4371         def _got_html_corrupt(res):
4372             self.failUnlessIn("Healthy : Healthy", res)
4373             self.failIfIn("Not Healthy", res)
4374             self.failUnlessIn("Repair successful", res)
4375         d.addCallback(_got_html_corrupt)
4376
4377         d.addErrback(self.explain_web_error)
4378         return d
4379
4380     def test_repair_json(self):
4381         self.basedir = "web/Grid/repair_json"
4382         self.set_up_grid()
4383         c0 = self.g.clients[0]
4384         self.uris = {}
4385         DATA = "data" * 100
4386         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4387         def _stash_uri(ur, which):
4388             self.uris[which] = ur.uri
4389         d.addCallback(_stash_uri, "sick")
4390
4391         def _compute_fileurls(ignored):
4392             self.fileurls = {}
4393             for which in self.uris:
4394                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4395         d.addCallback(_compute_fileurls)
4396
4397         def _clobber_shares(ignored):
4398             sick_shares = self.find_uri_shares(self.uris["sick"])
4399             os.unlink(sick_shares[0][2])
4400         d.addCallback(_clobber_shares)
4401
4402         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4403         def _got_json_sick(res):
4404             r = simplejson.loads(res)
4405             self.failUnlessReallyEqual(r["repair-attempted"], True)
4406             self.failUnlessReallyEqual(r["repair-successful"], True)
4407             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4408                                  "Not Healthy: 9 shares (enc 3-of-10)")
4409             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4410             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4411             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4412         d.addCallback(_got_json_sick)
4413
4414         d.addErrback(self.explain_web_error)
4415         return d
4416
4417     def test_unknown(self, immutable=False):
4418         self.basedir = "web/Grid/unknown"
4419         if immutable:
4420             self.basedir = "web/Grid/unknown-immutable"
4421
4422         self.set_up_grid()
4423         c0 = self.g.clients[0]
4424         self.uris = {}
4425         self.fileurls = {}
4426
4427         # the future cap format may contain slashes, which must be tolerated
4428         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4429                                                            safe="")
4430
4431         if immutable:
4432             name = u"future-imm"
4433             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4434             d = c0.create_immutable_dirnode({name: (future_node, {})})
4435         else:
4436             name = u"future"
4437             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4438             d = c0.create_dirnode()
4439
4440         def _stash_root_and_create_file(n):
4441             self.rootnode = n
4442             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4443             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4444             if not immutable:
4445                 return self.rootnode.set_node(name, future_node)
4446         d.addCallback(_stash_root_and_create_file)
4447
4448         # make sure directory listing tolerates unknown nodes
4449         d.addCallback(lambda ign: self.GET(self.rooturl))
4450         def _check_directory_html(res, expected_type_suffix):
4451             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4452                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4453                                  re.DOTALL)
4454             self.failUnless(re.search(pattern, res), res)
4455             # find the More Info link for name, should be relative
4456             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4457             info_url = mo.group(1)
4458             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4459         if immutable:
4460             d.addCallback(_check_directory_html, "-IMM")
4461         else:
4462             d.addCallback(_check_directory_html, "")
4463
4464         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4465         def _check_directory_json(res, expect_rw_uri):
4466             data = simplejson.loads(res)
4467             self.failUnlessEqual(data[0], "dirnode")
4468             f = data[1]["children"][name]
4469             self.failUnlessEqual(f[0], "unknown")
4470             if expect_rw_uri:
4471                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4472             else:
4473                 self.failIfIn("rw_uri", f[1])
4474             if immutable:
4475                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4476             else:
4477                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4478             self.failUnlessIn("metadata", f[1])
4479         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4480
4481         def _check_info(res, expect_rw_uri, expect_ro_uri):
4482             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4483             if expect_rw_uri:
4484                 self.failUnlessIn(unknown_rwcap, res)
4485             if expect_ro_uri:
4486                 if immutable:
4487                     self.failUnlessIn(unknown_immcap, res)
4488                 else:
4489                     self.failUnlessIn(unknown_rocap, res)
4490             else:
4491                 self.failIfIn(unknown_rocap, res)
4492             self.failIfIn("Raw data as", res)
4493             self.failIfIn("Directory writecap", res)
4494             self.failIfIn("Checker Operations", res)
4495             self.failIfIn("Mutable File Operations", res)
4496             self.failIfIn("Directory Operations", res)
4497
4498         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4499         # why they fail. Possibly related to ticket #922.
4500
4501         d.addCallback(lambda ign: self.GET(expected_info_url))
4502         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4503         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4504         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4505
4506         def _check_json(res, expect_rw_uri):
4507             data = simplejson.loads(res)
4508             self.failUnlessEqual(data[0], "unknown")
4509             if expect_rw_uri:
4510                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4511             else:
4512                 self.failIfIn("rw_uri", data[1])
4513
4514             if immutable:
4515                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4516                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4517             elif expect_rw_uri:
4518                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4519                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4520             else:
4521                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4522                 self.failIfIn("mutable", data[1])
4523
4524             # TODO: check metadata contents
4525             self.failUnlessIn("metadata", data[1])
4526
4527         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4528         d.addCallback(_check_json, expect_rw_uri=not immutable)
4529
4530         # and make sure that a read-only version of the directory can be
4531         # rendered too. This version will not have unknown_rwcap, whether
4532         # or not future_node was immutable.
4533         d.addCallback(lambda ign: self.GET(self.rourl))
4534         if immutable:
4535             d.addCallback(_check_directory_html, "-IMM")
4536         else:
4537             d.addCallback(_check_directory_html, "-RO")
4538
4539         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4540         d.addCallback(_check_directory_json, expect_rw_uri=False)
4541
4542         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4543         d.addCallback(_check_json, expect_rw_uri=False)
4544
4545         # TODO: check that getting t=info from the Info link in the ro directory
4546         # works, and does not include the writecap URI.
4547         return d
4548
4549     def test_immutable_unknown(self):
4550         return self.test_unknown(immutable=True)
4551
4552     def test_mutant_dirnodes_are_omitted(self):
4553         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4554
4555         self.set_up_grid()
4556         c = self.g.clients[0]
4557         nm = c.nodemaker
4558         self.uris = {}
4559         self.fileurls = {}
4560
4561         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4562         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4563         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4564
4565         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4566         # test the dirnode and web layers separately.
4567
4568         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4569         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4570         # When the directory is read, the mutants should be silently disposed of, leaving
4571         # their lonely sibling.
4572         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4573         # because immutable directories don't have a writecap and therefore that field
4574         # isn't (and can't be) decrypted.
4575         # TODO: The field still exists in the netstring. Technically we should check what
4576         # happens if something is put there (_unpack_contents should raise ValueError),
4577         # but that can wait.
4578
4579         lonely_child = nm.create_from_cap(lonely_uri)
4580         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4581         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4582
4583         def _by_hook_or_by_crook():
4584             return True
4585         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4586             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4587
4588         mutant_write_in_ro_child.get_write_uri    = lambda: None
4589         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4590
4591         kids = {u"lonely":      (lonely_child, {}),
4592                 u"ro":          (mutant_ro_child, {}),
4593                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4594                 }
4595         d = c.create_immutable_dirnode(kids)
4596
4597         def _created(dn):
4598             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4599             self.failIf(dn.is_mutable())
4600             self.failUnless(dn.is_readonly())
4601             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4602             self.failIf(hasattr(dn._node, 'get_writekey'))
4603             rep = str(dn)
4604             self.failUnlessIn("RO-IMM", rep)
4605             cap = dn.get_cap()
4606             self.failUnlessIn("CHK", cap.to_string())
4607             self.cap = cap
4608             self.rootnode = dn
4609             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4610             return download_to_data(dn._node)
4611         d.addCallback(_created)
4612
4613         def _check_data(data):
4614             # Decode the netstring representation of the directory to check that all children
4615             # are present. This is a bit of an abstraction violation, but there's not really
4616             # any other way to do it given that the real DirectoryNode._unpack_contents would
4617             # strip the mutant children out (which is what we're trying to test, later).
4618             position = 0
4619             numkids = 0
4620             while position < len(data):
4621                 entries, position = split_netstring(data, 1, position)
4622                 entry = entries[0]
4623                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4624                 name = name_utf8.decode("utf-8")
4625                 self.failUnlessEqual(rwcapdata, "")
4626                 self.failUnlessIn(name, kids)
4627                 (expected_child, ign) = kids[name]
4628                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4629                 numkids += 1
4630
4631             self.failUnlessReallyEqual(numkids, 3)
4632             return self.rootnode.list()
4633         d.addCallback(_check_data)
4634
4635         # Now when we use the real directory listing code, the mutants should be absent.
4636         def _check_kids(children):
4637             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4638             lonely_node, lonely_metadata = children[u"lonely"]
4639
4640             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4641             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4642         d.addCallback(_check_kids)
4643
4644         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4645         d.addCallback(lambda n: n.list())
4646         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4647
4648         # Make sure the lonely child can be listed in HTML...
4649         d.addCallback(lambda ign: self.GET(self.rooturl))
4650         def _check_html(res):
4651             self.failIfIn("URI:SSK", res)
4652             get_lonely = "".join([r'<td>FILE</td>',
4653                                   r'\s+<td>',
4654                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4655                                   r'</td>',
4656                                   r'\s+<td align="right">%d</td>' % len("one"),
4657                                   ])
4658             self.failUnless(re.search(get_lonely, res), res)
4659
4660             # find the More Info link for name, should be relative
4661             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4662             info_url = mo.group(1)
4663             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4664         d.addCallback(_check_html)
4665
4666         # ... and in JSON.
4667         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4668         def _check_json(res):
4669             data = simplejson.loads(res)
4670             self.failUnlessEqual(data[0], "dirnode")
4671             listed_children = data[1]["children"]
4672             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4673             ll_type, ll_data = listed_children[u"lonely"]
4674             self.failUnlessEqual(ll_type, "filenode")
4675             self.failIfIn("rw_uri", ll_data)
4676             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4677         d.addCallback(_check_json)
4678         return d
4679
4680     def test_deep_check(self):
4681         self.basedir = "web/Grid/deep_check"
4682         self.set_up_grid()
4683         c0 = self.g.clients[0]
4684         self.uris = {}
4685         self.fileurls = {}
4686         DATA = "data" * 100
4687         d = c0.create_dirnode()
4688         def _stash_root_and_create_file(n):
4689             self.rootnode = n
4690             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4691             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4692         d.addCallback(_stash_root_and_create_file)
4693         def _stash_uri(fn, which):
4694             self.uris[which] = fn.get_uri()
4695             return fn
4696         d.addCallback(_stash_uri, "good")
4697         d.addCallback(lambda ign:
4698                       self.rootnode.add_file(u"small",
4699                                              upload.Data("literal",
4700                                                         convergence="")))
4701         d.addCallback(_stash_uri, "small")
4702         d.addCallback(lambda ign:
4703                       self.rootnode.add_file(u"sick",
4704                                              upload.Data(DATA+"1",
4705                                                         convergence="")))
4706         d.addCallback(_stash_uri, "sick")
4707
4708         # this tests that deep-check and stream-manifest will ignore
4709         # UnknownNode instances. Hopefully this will also cover deep-stats.
4710         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4711         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4712
4713         def _clobber_shares(ignored):
4714             self.delete_shares_numbered(self.uris["sick"], [0,1])
4715         d.addCallback(_clobber_shares)
4716
4717         # root
4718         # root/good
4719         # root/small
4720         # root/sick
4721         # root/future
4722
4723         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4724         def _done(res):
4725             try:
4726                 units = [simplejson.loads(line)
4727                          for line in res.splitlines()
4728                          if line]
4729             except ValueError:
4730                 print "response is:", res
4731                 print "undecodeable line was '%s'" % line
4732                 raise
4733             self.failUnlessReallyEqual(len(units), 5+1)
4734             # should be parent-first
4735             u0 = units[0]
4736             self.failUnlessEqual(u0["path"], [])
4737             self.failUnlessEqual(u0["type"], "directory")
4738             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4739             u0cr = u0["check-results"]
4740             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4741
4742             ugood = [u for u in units
4743                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4744             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4745             ugoodcr = ugood["check-results"]
4746             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4747
4748             stats = units[-1]
4749             self.failUnlessEqual(stats["type"], "stats")
4750             s = stats["stats"]
4751             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4752             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4753             self.failUnlessReallyEqual(s["count-directories"], 1)
4754             self.failUnlessReallyEqual(s["count-unknown"], 1)
4755         d.addCallback(_done)
4756
4757         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4758         def _check_manifest(res):
4759             self.failUnless(res.endswith("\n"))
4760             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4761             self.failUnlessReallyEqual(len(units), 5+1)
4762             self.failUnlessEqual(units[-1]["type"], "stats")
4763             first = units[0]
4764             self.failUnlessEqual(first["path"], [])
4765             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4766             self.failUnlessEqual(first["type"], "directory")
4767             stats = units[-1]["stats"]
4768             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4769             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4770             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4771             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4772             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4773         d.addCallback(_check_manifest)
4774
4775         # now add root/subdir and root/subdir/grandchild, then make subdir
4776         # unrecoverable, then see what happens
4777
4778         d.addCallback(lambda ign:
4779                       self.rootnode.create_subdirectory(u"subdir"))
4780         d.addCallback(_stash_uri, "subdir")
4781         d.addCallback(lambda subdir_node:
4782                       subdir_node.add_file(u"grandchild",
4783                                            upload.Data(DATA+"2",
4784                                                        convergence="")))
4785         d.addCallback(_stash_uri, "grandchild")
4786
4787         d.addCallback(lambda ign:
4788                       self.delete_shares_numbered(self.uris["subdir"],
4789                                                   range(1, 10)))
4790
4791         # root
4792         # root/good
4793         # root/small
4794         # root/sick
4795         # root/future
4796         # root/subdir [unrecoverable]
4797         # root/subdir/grandchild
4798
4799         # how should a streaming-JSON API indicate fatal error?
4800         # answer: emit ERROR: instead of a JSON string
4801
4802         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4803         def _check_broken_manifest(res):
4804             lines = res.splitlines()
4805             error_lines = [i
4806                            for (i,line) in enumerate(lines)
4807                            if line.startswith("ERROR:")]
4808             if not error_lines:
4809                 self.fail("no ERROR: in output: %s" % (res,))
4810             first_error = error_lines[0]
4811             error_line = lines[first_error]
4812             error_msg = lines[first_error+1:]
4813             error_msg_s = "\n".join(error_msg) + "\n"
4814             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4815                               error_line)
4816             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4817             units = [simplejson.loads(line) for line in lines[:first_error]]
4818             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4819             last_unit = units[-1]
4820             self.failUnlessEqual(last_unit["path"], ["subdir"])
4821         d.addCallback(_check_broken_manifest)
4822
4823         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4824         def _check_broken_deepcheck(res):
4825             lines = res.splitlines()
4826             error_lines = [i
4827                            for (i,line) in enumerate(lines)
4828                            if line.startswith("ERROR:")]
4829             if not error_lines:
4830                 self.fail("no ERROR: in output: %s" % (res,))
4831             first_error = error_lines[0]
4832             error_line = lines[first_error]
4833             error_msg = lines[first_error+1:]
4834             error_msg_s = "\n".join(error_msg) + "\n"
4835             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4836                               error_line)
4837             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4838             units = [simplejson.loads(line) for line in lines[:first_error]]
4839             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4840             last_unit = units[-1]
4841             self.failUnlessEqual(last_unit["path"], ["subdir"])
4842             r = last_unit["check-results"]["results"]
4843             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4844             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4845             self.failUnlessReallyEqual(r["recoverable"], False)
4846         d.addCallback(_check_broken_deepcheck)
4847
4848         d.addErrback(self.explain_web_error)
4849         return d
4850
4851     def test_deep_check_and_repair(self):
4852         self.basedir = "web/Grid/deep_check_and_repair"
4853         self.set_up_grid()
4854         c0 = self.g.clients[0]
4855         self.uris = {}
4856         self.fileurls = {}
4857         DATA = "data" * 100
4858         d = c0.create_dirnode()
4859         def _stash_root_and_create_file(n):
4860             self.rootnode = n
4861             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4862             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4863         d.addCallback(_stash_root_and_create_file)
4864         def _stash_uri(fn, which):
4865             self.uris[which] = fn.get_uri()
4866         d.addCallback(_stash_uri, "good")
4867         d.addCallback(lambda ign:
4868                       self.rootnode.add_file(u"small",
4869                                              upload.Data("literal",
4870                                                         convergence="")))
4871         d.addCallback(_stash_uri, "small")
4872         d.addCallback(lambda ign:
4873                       self.rootnode.add_file(u"sick",
4874                                              upload.Data(DATA+"1",
4875                                                         convergence="")))
4876         d.addCallback(_stash_uri, "sick")
4877         #d.addCallback(lambda ign:
4878         #              self.rootnode.add_file(u"dead",
4879         #                                     upload.Data(DATA+"2",
4880         #                                                convergence="")))
4881         #d.addCallback(_stash_uri, "dead")
4882
4883         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4884         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4885         #d.addCallback(_stash_uri, "corrupt")
4886
4887         def _clobber_shares(ignored):
4888             good_shares = self.find_uri_shares(self.uris["good"])
4889             self.failUnlessReallyEqual(len(good_shares), 10)
4890             sick_shares = self.find_uri_shares(self.uris["sick"])
4891             os.unlink(sick_shares[0][2])
4892             #dead_shares = self.find_uri_shares(self.uris["dead"])
4893             #for i in range(1, 10):
4894             #    os.unlink(dead_shares[i][2])
4895
4896             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4897             #cso = CorruptShareOptions()
4898             #cso.stdout = StringIO()
4899             #cso.parseOptions([c_shares[0][2]])
4900             #corrupt_share(cso)
4901         d.addCallback(_clobber_shares)
4902
4903         # root
4904         # root/good   CHK, 10 shares
4905         # root/small  LIT
4906         # root/sick   CHK, 9 shares
4907
4908         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4909         def _done(res):
4910             units = [simplejson.loads(line)
4911                      for line in res.splitlines()
4912                      if line]
4913             self.failUnlessReallyEqual(len(units), 4+1)
4914             # should be parent-first
4915             u0 = units[0]
4916             self.failUnlessEqual(u0["path"], [])
4917             self.failUnlessEqual(u0["type"], "directory")
4918             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4919             u0crr = u0["check-and-repair-results"]
4920             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4921             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4922
4923             ugood = [u for u in units
4924                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4925             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4926             ugoodcrr = ugood["check-and-repair-results"]
4927             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4928             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4929
4930             usick = [u for u in units
4931                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4932             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4933             usickcrr = usick["check-and-repair-results"]
4934             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4935             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4936             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4937             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4938
4939             stats = units[-1]
4940             self.failUnlessEqual(stats["type"], "stats")
4941             s = stats["stats"]
4942             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4943             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4944             self.failUnlessReallyEqual(s["count-directories"], 1)
4945         d.addCallback(_done)
4946
4947         d.addErrback(self.explain_web_error)
4948         return d
4949
4950     def _count_leases(self, ignored, which):
4951         u = self.uris[which]
4952         shares = self.find_uri_shares(u)
4953         lease_counts = []
4954         for shnum, serverid, fn in shares:
4955             sf = get_share_file(fn)
4956             num_leases = len(list(sf.get_leases()))
4957             lease_counts.append( (fn, num_leases) )
4958         return lease_counts
4959
4960     def _assert_leasecount(self, lease_counts, expected):
4961         for (fn, num_leases) in lease_counts:
4962             if num_leases != expected:
4963                 self.fail("expected %d leases, have %d, on %s" %
4964                           (expected, num_leases, fn))
4965
4966     def test_add_lease(self):
4967         self.basedir = "web/Grid/add_lease"
4968         self.set_up_grid(num_clients=2)
4969         c0 = self.g.clients[0]
4970         self.uris = {}
4971         DATA = "data" * 100
4972         d = c0.upload(upload.Data(DATA, convergence=""))
4973         def _stash_uri(ur, which):
4974             self.uris[which] = ur.uri
4975         d.addCallback(_stash_uri, "one")
4976         d.addCallback(lambda ign:
4977                       c0.upload(upload.Data(DATA+"1", convergence="")))
4978         d.addCallback(_stash_uri, "two")
4979         def _stash_mutable_uri(n, which):
4980             self.uris[which] = n.get_uri()
4981             assert isinstance(self.uris[which], str)
4982         d.addCallback(lambda ign:
4983             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4984         d.addCallback(_stash_mutable_uri, "mutable")
4985
4986         def _compute_fileurls(ignored):
4987             self.fileurls = {}
4988             for which in self.uris:
4989                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4990         d.addCallback(_compute_fileurls)
4991
4992         d.addCallback(self._count_leases, "one")
4993         d.addCallback(self._assert_leasecount, 1)
4994         d.addCallback(self._count_leases, "two")
4995         d.addCallback(self._assert_leasecount, 1)
4996         d.addCallback(self._count_leases, "mutable")
4997         d.addCallback(self._assert_leasecount, 1)
4998
4999         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5000         def _got_html_good(res):
5001             self.failUnlessIn("Healthy", res)
5002             self.failIfIn("Not Healthy", res)
5003         d.addCallback(_got_html_good)
5004
5005         d.addCallback(self._count_leases, "one")
5006         d.addCallback(self._assert_leasecount, 1)
5007         d.addCallback(self._count_leases, "two")
5008         d.addCallback(self._assert_leasecount, 1)
5009         d.addCallback(self._count_leases, "mutable")
5010         d.addCallback(self._assert_leasecount, 1)
5011
5012         # this CHECK uses the original client, which uses the same
5013         # lease-secrets, so it will just renew the original lease
5014         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5015         d.addCallback(_got_html_good)
5016
5017         d.addCallback(self._count_leases, "one")
5018         d.addCallback(self._assert_leasecount, 1)
5019         d.addCallback(self._count_leases, "two")
5020         d.addCallback(self._assert_leasecount, 1)
5021         d.addCallback(self._count_leases, "mutable")
5022         d.addCallback(self._assert_leasecount, 1)
5023
5024         # this CHECK uses an alternate client, which adds a second lease
5025         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5026         d.addCallback(_got_html_good)
5027
5028         d.addCallback(self._count_leases, "one")
5029         d.addCallback(self._assert_leasecount, 2)
5030         d.addCallback(self._count_leases, "two")
5031         d.addCallback(self._assert_leasecount, 1)
5032         d.addCallback(self._count_leases, "mutable")
5033         d.addCallback(self._assert_leasecount, 1)
5034
5035         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5036         d.addCallback(_got_html_good)
5037
5038         d.addCallback(self._count_leases, "one")
5039         d.addCallback(self._assert_leasecount, 2)
5040         d.addCallback(self._count_leases, "two")
5041         d.addCallback(self._assert_leasecount, 1)
5042         d.addCallback(self._count_leases, "mutable")
5043         d.addCallback(self._assert_leasecount, 1)
5044
5045         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5046                       clientnum=1)
5047         d.addCallback(_got_html_good)
5048
5049         d.addCallback(self._count_leases, "one")
5050         d.addCallback(self._assert_leasecount, 2)
5051         d.addCallback(self._count_leases, "two")
5052         d.addCallback(self._assert_leasecount, 1)
5053         d.addCallback(self._count_leases, "mutable")
5054         d.addCallback(self._assert_leasecount, 2)
5055
5056         d.addErrback(self.explain_web_error)
5057         return d
5058
5059     def test_deep_add_lease(self):
5060         self.basedir = "web/Grid/deep_add_lease"
5061         self.set_up_grid(num_clients=2)
5062         c0 = self.g.clients[0]
5063         self.uris = {}
5064         self.fileurls = {}
5065         DATA = "data" * 100
5066         d = c0.create_dirnode()
5067         def _stash_root_and_create_file(n):
5068             self.rootnode = n
5069             self.uris["root"] = n.get_uri()
5070             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5071             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5072         d.addCallback(_stash_root_and_create_file)
5073         def _stash_uri(fn, which):
5074             self.uris[which] = fn.get_uri()
5075         d.addCallback(_stash_uri, "one")
5076         d.addCallback(lambda ign:
5077                       self.rootnode.add_file(u"small",
5078                                              upload.Data("literal",
5079                                                         convergence="")))
5080         d.addCallback(_stash_uri, "small")
5081
5082         d.addCallback(lambda ign:
5083             c0.create_mutable_file(publish.MutableData("mutable")))
5084         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5085         d.addCallback(_stash_uri, "mutable")
5086
5087         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5088         def _done(res):
5089             units = [simplejson.loads(line)
5090                      for line in res.splitlines()
5091                      if line]
5092             # root, one, small, mutable,   stats
5093             self.failUnlessReallyEqual(len(units), 4+1)
5094         d.addCallback(_done)
5095
5096         d.addCallback(self._count_leases, "root")
5097         d.addCallback(self._assert_leasecount, 1)
5098         d.addCallback(self._count_leases, "one")
5099         d.addCallback(self._assert_leasecount, 1)
5100         d.addCallback(self._count_leases, "mutable")
5101         d.addCallback(self._assert_leasecount, 1)
5102
5103         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5104         d.addCallback(_done)
5105
5106         d.addCallback(self._count_leases, "root")
5107         d.addCallback(self._assert_leasecount, 1)
5108         d.addCallback(self._count_leases, "one")
5109         d.addCallback(self._assert_leasecount, 1)
5110         d.addCallback(self._count_leases, "mutable")
5111         d.addCallback(self._assert_leasecount, 1)
5112
5113         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5114                       clientnum=1)
5115         d.addCallback(_done)
5116
5117         d.addCallback(self._count_leases, "root")
5118         d.addCallback(self._assert_leasecount, 2)
5119         d.addCallback(self._count_leases, "one")
5120         d.addCallback(self._assert_leasecount, 2)
5121         d.addCallback(self._count_leases, "mutable")
5122         d.addCallback(self._assert_leasecount, 2)
5123
5124         d.addErrback(self.explain_web_error)
5125         return d
5126
5127
5128     def test_exceptions(self):
5129         self.basedir = "web/Grid/exceptions"
5130         self.set_up_grid(num_clients=1, num_servers=2)
5131         c0 = self.g.clients[0]
5132         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5133         self.fileurls = {}
5134         DATA = "data" * 100
5135         d = c0.create_dirnode()
5136         def _stash_root(n):
5137             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5138             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5139             return n
5140         d.addCallback(_stash_root)
5141         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5142         def _stash_bad(ur):
5143             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5144             self.delete_shares_numbered(ur.uri, range(1,10))
5145
5146             u = uri.from_string(ur.uri)
5147             u.key = testutil.flip_bit(u.key, 0)
5148             baduri = u.to_string()
5149             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5150         d.addCallback(_stash_bad)
5151         d.addCallback(lambda ign: c0.create_dirnode())
5152         def _mangle_dirnode_1share(n):
5153             u = n.get_uri()
5154             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5155             self.fileurls["dir-1share-json"] = url + "?t=json"
5156             self.delete_shares_numbered(u, range(1,10))
5157         d.addCallback(_mangle_dirnode_1share)
5158         d.addCallback(lambda ign: c0.create_dirnode())
5159         def _mangle_dirnode_0share(n):
5160             u = n.get_uri()
5161             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5162             self.fileurls["dir-0share-json"] = url + "?t=json"
5163             self.delete_shares_numbered(u, range(0,10))
5164         d.addCallback(_mangle_dirnode_0share)
5165
5166         # NotEnoughSharesError should be reported sensibly, with a
5167         # text/plain explanation of the problem, and perhaps some
5168         # information on which shares *could* be found.
5169
5170         d.addCallback(lambda ignored:
5171                       self.shouldHTTPError("GET unrecoverable",
5172                                            410, "Gone", "NoSharesError",
5173                                            self.GET, self.fileurls["0shares"]))
5174         def _check_zero_shares(body):
5175             self.failIfIn("<html>", body)
5176             body = " ".join(body.strip().split())
5177             exp = ("NoSharesError: no shares could be found. "
5178                    "Zero shares usually indicates a corrupt URI, or that "
5179                    "no servers were connected, but it might also indicate "
5180                    "severe corruption. You should perform a filecheck on "
5181                    "this object to learn more. The full error message is: "
5182                    "no shares (need 3). Last failure: None")
5183             self.failUnlessReallyEqual(exp, body)
5184         d.addCallback(_check_zero_shares)
5185
5186
5187         d.addCallback(lambda ignored:
5188                       self.shouldHTTPError("GET 1share",
5189                                            410, "Gone", "NotEnoughSharesError",
5190                                            self.GET, self.fileurls["1share"]))
5191         def _check_one_share(body):
5192             self.failIfIn("<html>", body)
5193             body = " ".join(body.strip().split())
5194             msgbase = ("NotEnoughSharesError: This indicates that some "
5195                        "servers were unavailable, or that shares have been "
5196                        "lost to server departure, hard drive failure, or disk "
5197                        "corruption. You should perform a filecheck on "
5198                        "this object to learn more. The full error message is:"
5199                        )
5200             msg1 = msgbase + (" ran out of shares:"
5201                               " complete=sh0"
5202                               " pending="
5203                               " overdue= unused= need 3. Last failure: None")
5204             msg2 = msgbase + (" ran out of shares:"
5205                               " complete="
5206                               " pending=Share(sh0-on-xgru5)"
5207                               " overdue= unused= need 3. Last failure: None")
5208             self.failUnless(body == msg1 or body == msg2, body)
5209         d.addCallback(_check_one_share)
5210
5211         d.addCallback(lambda ignored:
5212                       self.shouldHTTPError("GET imaginary",
5213                                            404, "Not Found", None,
5214                                            self.GET, self.fileurls["imaginary"]))
5215         def _missing_child(body):
5216             self.failUnlessIn("No such child: imaginary", body)
5217         d.addCallback(_missing_child)
5218
5219         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5220         def _check_0shares_dir_html(body):
5221             self.failUnlessIn("<html>", body)
5222             # we should see the regular page, but without the child table or
5223             # the dirops forms
5224             body = " ".join(body.strip().split())
5225             self.failUnlessIn('href="?t=info">More info on this directory',
5226                               body)
5227             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5228                    "could not be retrieved, because there were insufficient "
5229                    "good shares. This might indicate that no servers were "
5230                    "connected, insufficient servers were connected, the URI "
5231                    "was corrupt, or that shares have been lost due to server "
5232                    "departure, hard drive failure, or disk corruption. You "
5233                    "should perform a filecheck on this object to learn more.")
5234             self.failUnlessIn(exp, body)
5235             self.failUnlessIn("No upload forms: directory is unreadable", body)
5236         d.addCallback(_check_0shares_dir_html)
5237
5238         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5239         def _check_1shares_dir_html(body):
5240             # at some point, we'll split UnrecoverableFileError into 0-shares
5241             # and some-shares like we did for immutable files (since there
5242             # are different sorts of advice to offer in each case). For now,
5243             # they present the same way.
5244             self.failUnlessIn("<html>", body)
5245             body = " ".join(body.strip().split())
5246             self.failUnlessIn('href="?t=info">More info on this directory',
5247                               body)
5248             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5249                    "could not be retrieved, because there were insufficient "
5250                    "good shares. This might indicate that no servers were "
5251                    "connected, insufficient servers were connected, the URI "
5252                    "was corrupt, or that shares have been lost due to server "
5253                    "departure, hard drive failure, or disk corruption. You "
5254                    "should perform a filecheck on this object to learn more.")
5255             self.failUnlessIn(exp, body)
5256             self.failUnlessIn("No upload forms: directory is unreadable", body)
5257         d.addCallback(_check_1shares_dir_html)
5258
5259         d.addCallback(lambda ignored:
5260                       self.shouldHTTPError("GET dir-0share-json",
5261                                            410, "Gone", "UnrecoverableFileError",
5262                                            self.GET,
5263                                            self.fileurls["dir-0share-json"]))
5264         def _check_unrecoverable_file(body):
5265             self.failIfIn("<html>", body)
5266             body = " ".join(body.strip().split())
5267             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5268                    "could not be retrieved, because there were insufficient "
5269                    "good shares. This might indicate that no servers were "
5270                    "connected, insufficient servers were connected, the URI "
5271                    "was corrupt, or that shares have been lost due to server "
5272                    "departure, hard drive failure, or disk corruption. You "
5273                    "should perform a filecheck on this object to learn more.")
5274             self.failUnlessReallyEqual(exp, body)
5275         d.addCallback(_check_unrecoverable_file)
5276
5277         d.addCallback(lambda ignored:
5278                       self.shouldHTTPError("GET dir-1share-json",
5279                                            410, "Gone", "UnrecoverableFileError",
5280                                            self.GET,
5281                                            self.fileurls["dir-1share-json"]))
5282         d.addCallback(_check_unrecoverable_file)
5283
5284         d.addCallback(lambda ignored:
5285                       self.shouldHTTPError("GET imaginary",
5286                                            404, "Not Found", None,
5287                                            self.GET, self.fileurls["imaginary"]))
5288
5289         # attach a webapi child that throws a random error, to test how it
5290         # gets rendered.
5291         w = c0.getServiceNamed("webish")
5292         w.root.putChild("ERRORBOOM", ErrorBoom())
5293
5294         # "Accept: */*" :        should get a text/html stack trace
5295         # "Accept: text/plain" : should get a text/plain stack trace
5296         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5297         # no Accept header:      should get a text/html stack trace
5298
5299         d.addCallback(lambda ignored:
5300                       self.shouldHTTPError("GET errorboom_html",
5301                                            500, "Internal Server Error", None,
5302                                            self.GET, "ERRORBOOM",
5303                                            headers={"accept": ["*/*"]}))
5304         def _internal_error_html1(body):
5305             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5306         d.addCallback(_internal_error_html1)
5307
5308         d.addCallback(lambda ignored:
5309                       self.shouldHTTPError("GET errorboom_text",
5310                                            500, "Internal Server Error", None,
5311                                            self.GET, "ERRORBOOM",
5312                                            headers={"accept": ["text/plain"]}))
5313         def _internal_error_text2(body):
5314             self.failIfIn("<html>", body)
5315             self.failUnless(body.startswith("Traceback "), body)
5316         d.addCallback(_internal_error_text2)
5317
5318         CLI_accepts = "text/plain, application/octet-stream"
5319         d.addCallback(lambda ignored:
5320                       self.shouldHTTPError("GET errorboom_text",
5321                                            500, "Internal Server Error", None,
5322                                            self.GET, "ERRORBOOM",
5323                                            headers={"accept": [CLI_accepts]}))
5324         def _internal_error_text3(body):
5325             self.failIfIn("<html>", body)
5326             self.failUnless(body.startswith("Traceback "), body)
5327         d.addCallback(_internal_error_text3)
5328
5329         d.addCallback(lambda ignored:
5330                       self.shouldHTTPError("GET errorboom_text",
5331                                            500, "Internal Server Error", None,
5332                                            self.GET, "ERRORBOOM"))
5333         def _internal_error_html4(body):
5334             self.failUnlessIn("<html>", body)
5335         d.addCallback(_internal_error_html4)
5336
5337         def _flush_errors(res):
5338             # Trial: please ignore the CompletelyUnhandledError in the logs
5339             self.flushLoggedErrors(CompletelyUnhandledError)
5340             return res
5341         d.addBoth(_flush_errors)
5342
5343         return d
5344
5345     def test_blacklist(self):
5346         # download from a blacklisted URI, get an error
5347         self.basedir = "web/Grid/blacklist"
5348         self.set_up_grid()
5349         c0 = self.g.clients[0]
5350         c0_basedir = c0.basedir
5351         fn = os.path.join(c0_basedir, "access.blacklist")
5352         self.uris = {}
5353         DATA = "off-limits " * 50
5354
5355         d = c0.upload(upload.Data(DATA, convergence=""))
5356         def _stash_uri_and_create_dir(ur):
5357             self.uri = ur.uri
5358             self.url = "uri/"+self.uri
5359             u = uri.from_string_filenode(self.uri)
5360             self.si = u.get_storage_index()
5361             childnode = c0.create_node_from_uri(self.uri, None)
5362             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5363         d.addCallback(_stash_uri_and_create_dir)
5364         def _stash_dir(node):
5365             self.dir_node = node
5366             self.dir_uri = node.get_uri()
5367             self.dir_url = "uri/"+self.dir_uri
5368         d.addCallback(_stash_dir)
5369         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5370         def _check_dir_html(body):
5371             self.failUnlessIn("<html>", body)
5372             self.failUnlessIn("blacklisted.txt</a>", body)
5373         d.addCallback(_check_dir_html)
5374         d.addCallback(lambda ign: self.GET(self.url))
5375         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5376
5377         def _blacklist(ign):
5378             f = open(fn, "w")
5379             f.write(" # this is a comment\n")
5380             f.write(" \n")
5381             f.write("\n") # also exercise blank lines
5382             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5383             f.close()
5384             # clients should be checking the blacklist each time, so we don't
5385             # need to restart the client
5386         d.addCallback(_blacklist)
5387         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5388                                                        403, "Forbidden",
5389                                                        "Access Prohibited: off-limits",
5390                                                        self.GET, self.url))
5391
5392         # We should still be able to list the parent directory, in HTML...
5393         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5394         def _check_dir_html2(body):
5395             self.failUnlessIn("<html>", body)
5396             self.failUnlessIn("blacklisted.txt</strike>", body)
5397         d.addCallback(_check_dir_html2)
5398
5399         # ... and in JSON (used by CLI).
5400         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5401         def _check_dir_json(res):
5402             data = simplejson.loads(res)
5403             self.failUnless(isinstance(data, list), data)
5404             self.failUnlessEqual(data[0], "dirnode")
5405             self.failUnless(isinstance(data[1], dict), data)
5406             self.failUnlessIn("children", data[1])
5407             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5408             childdata = data[1]["children"]["blacklisted.txt"]
5409             self.failUnless(isinstance(childdata, list), data)
5410             self.failUnlessEqual(childdata[0], "filenode")
5411             self.failUnless(isinstance(childdata[1], dict), data)
5412         d.addCallback(_check_dir_json)
5413
5414         def _unblacklist(ign):
5415             open(fn, "w").close()
5416             # the Blacklist object watches mtime to tell when the file has
5417             # changed, but on windows this test will run faster than the
5418             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5419             # to force a reload.
5420             self.g.clients[0].blacklist.last_mtime -= 2.0
5421         d.addCallback(_unblacklist)
5422
5423         # now a read should work
5424         d.addCallback(lambda ign: self.GET(self.url))
5425         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5426
5427         # read again to exercise the blacklist-is-unchanged logic
5428         d.addCallback(lambda ign: self.GET(self.url))
5429         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5430
5431         # now add a blacklisted directory, and make sure files under it are
5432         # refused too
5433         def _add_dir(ign):
5434             childnode = c0.create_node_from_uri(self.uri, None)
5435             return c0.create_dirnode({u"child": (childnode,{}) })
5436         d.addCallback(_add_dir)
5437         def _get_dircap(dn):
5438             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5439             self.dir_url_base = "uri/"+dn.get_write_uri()
5440             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5441             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5442             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5443             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5444         d.addCallback(_get_dircap)
5445         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5446         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5447         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5448         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5449         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5450         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5451         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5452         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5453         d.addCallback(lambda ign: self.GET(self.child_url))
5454         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5455
5456         def _block_dir(ign):
5457             f = open(fn, "w")
5458             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5459             f.close()
5460             self.g.clients[0].blacklist.last_mtime -= 2.0
5461         d.addCallback(_block_dir)
5462         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5463                                                        403, "Forbidden",
5464                                                        "Access Prohibited: dir-off-limits",
5465                                                        self.GET, self.dir_url_base))
5466         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5467                                                        403, "Forbidden",
5468                                                        "Access Prohibited: dir-off-limits",
5469                                                        self.GET, self.dir_url_json1))
5470         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5471                                                        403, "Forbidden",
5472                                                        "Access Prohibited: dir-off-limits",
5473                                                        self.GET, self.dir_url_json2))
5474         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5475                                                        403, "Forbidden",
5476                                                        "Access Prohibited: dir-off-limits",
5477                                                        self.GET, self.dir_url_json_ro))
5478         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5479                                                        403, "Forbidden",
5480                                                        "Access Prohibited: dir-off-limits",
5481                                                        self.GET, self.child_url))
5482         return d
5483
5484
5485 class CompletelyUnhandledError(Exception):
5486     pass
5487 class ErrorBoom(rend.Page):
5488     def beforeRender(self, ctx):
5489         raise CompletelyUnhandledError("whoops")