src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     # TODO: version of this with a Unicode filename
 904     def test_GET_FILEURL_save(self):
 905         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 906                      return_response=True)
 907         def _got((res, statuscode, headers)):
 908             content_disposition = headers["content-disposition"][0]
 909             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 910             self.failUnlessIsBarDotTxt(res)
 911         d.addCallback(_got)
 912         return d
 913
 914     def test_GET_FILEURL_missing(self):
 915         d = self.GET(self.public_url + "/foo/missing")
 916         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 917         return d
 918
 919     def test_GET_FILEURL_info_mdmf(self):
 920         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 921         def _got(res):
 922             self.failUnlessIn("mutable file (mdmf)", res)
 923             self.failUnlessIn(self._quux_txt_uri, res)
 924             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_GET_FILEURL_info_mdmf_readonly(self):
 929         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 930         def _got(res):
 931             self.failUnlessIn("mutable file (mdmf)", res)
 932             self.failIfIn(self._quux_txt_uri, res)
 933             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 934         d.addCallback(_got)
 935         return d
 936
 937     def test_GET_FILEURL_info_sdmf(self):
 938         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 939         def _got(res):
 940             self.failUnlessIn("mutable file (sdmf)", res)
 941             self.failUnlessIn(self._baz_txt_uri, res)
 942         d.addCallback(_got)
 943         return d
 944
 945     def test_GET_FILEURL_info_mdmf_extensions(self):
 946         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 947         def _got(res):
 948             self.failUnlessIn("mutable file (mdmf)", res)
 949             self.failUnlessIn(self._quux_txt_uri, res)
 950             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 951         d.addCallback(_got)
 952         return d
 953
 954     def test_PUT_overwrite_only_files(self):
 955         # create a directory, put a file in that directory.
 956         contents, n, filecap = self.makefile(8)
 957         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt",
 960                      self.NEWFILE_CONTENTS))
 961         # try to overwrite the file with replace=only-files
 962         # (this should work)
 963         d.addCallback(lambda res:
 964             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 965                      filecap))
 966         d.addCallback(lambda res:
 967             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 968                  "There was already a child by that name, and you asked me "
 969                  "to not replace it",
 970                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 971                  filecap))
 972         return d
 973
 974     def test_PUT_NEWFILEURL(self):
 975         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 976         # TODO: we lose the response code, so we can't check this
 977         #self.failUnlessReallyEqual(responsecode, 201)
 978         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 979         d.addCallback(lambda res:
 980                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 981                                                       self.NEWFILE_CONTENTS))
 982         return d
 983
 984     def test_PUT_NEWFILEURL_not_mutable(self):
 985         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 986                      self.NEWFILE_CONTENTS)
 987         # TODO: we lose the response code, so we can't check this
 988         #self.failUnlessReallyEqual(responsecode, 201)
 989         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 990         d.addCallback(lambda res:
 991                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 992                                                       self.NEWFILE_CONTENTS))
 993         return d
 994
 995     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 996         # this should get us a few segments of an MDMF mutable file,
 997         # which we can then test for.
 998         contents = self.NEWFILE_CONTENTS * 300000
 999         d = self.PUT("/uri?format=mdmf",
1000                      contents)
1001         def _got_filecap(filecap):
1002             self.failUnless(filecap.startswith("URI:MDMF"))
1003             return filecap
1004         d.addCallback(_got_filecap)
1005         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1006         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1007         return d
1008
1009     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1010         contents = self.NEWFILE_CONTENTS * 300000
1011         d = self.PUT("/uri?format=sdmf",
1012                      contents)
1013         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1014         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1015         return d
1016
1017     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1018         contents = self.NEWFILE_CONTENTS * 300000
1019         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1020                                     400, "Bad Request", "Unknown format: foo",
1021                                     self.PUT, "/uri?format=foo",
1022                                     contents)
1023
1024     def test_PUT_NEWFILEURL_range_bad(self):
1025         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1026         target = self.public_url + "/foo/new.txt"
1027         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1028                              "501 Not Implemented",
1029                              "Content-Range in PUT not yet supported",
1030                              # (and certainly not for immutable files)
1031                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1032                              headers=headers)
1033         d.addCallback(lambda res:
1034                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1035         return d
1036
1037     def test_PUT_NEWFILEURL_mutable(self):
1038         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1039                      self.NEWFILE_CONTENTS)
1040         # TODO: we lose the response code, so we can't check this
1041         #self.failUnlessReallyEqual(responsecode, 201)
1042         def _check_uri(res):
1043             u = uri.from_string_mutable_filenode(res)
1044             self.failUnless(u.is_mutable())
1045             self.failIf(u.is_readonly())
1046             return res
1047         d.addCallback(_check_uri)
1048         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1049         d.addCallback(lambda res:
1050                       self.failUnlessMutableChildContentsAre(self._foo_node,
1051                                                              u"new.txt",
1052                                                              self.NEWFILE_CONTENTS))
1053         return d
1054
1055     def test_PUT_NEWFILEURL_mutable_toobig(self):
1056         # It is okay to upload large mutable files, so we should be able
1057         # to do that.
1058         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1059                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1060         return d
1061
1062     def test_PUT_NEWFILEURL_replace(self):
1063         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1064         # TODO: we lose the response code, so we can't check this
1065         #self.failUnlessReallyEqual(responsecode, 200)
1066         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1067         d.addCallback(lambda res:
1068                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1069                                                       self.NEWFILE_CONTENTS))
1070         return d
1071
1072     def test_PUT_NEWFILEURL_bad_t(self):
1073         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1074                              "PUT to a file: bad t=bogus",
1075                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1076                              "contents")
1077         return d
1078
1079     def test_PUT_NEWFILEURL_no_replace(self):
1080         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1081                      self.NEWFILE_CONTENTS)
1082         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1083                   "409 Conflict",
1084                   "There was already a child by that name, and you asked me "
1085                   "to not replace it")
1086         return d
1087
1088     def test_PUT_NEWFILEURL_mkdirs(self):
1089         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1090         fn = self._foo_node
1091         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1092         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1093         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1094         d.addCallback(lambda res:
1095                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1096                                                       self.NEWFILE_CONTENTS))
1097         return d
1098
1099     def test_PUT_NEWFILEURL_blocked(self):
1100         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1101                      self.NEWFILE_CONTENTS)
1102         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1103                   "409 Conflict",
1104                   "Unable to create directory 'blockingfile': a file was in the way")
1105         return d
1106
1107     def test_PUT_NEWFILEURL_emptyname(self):
1108         # an empty pathname component (i.e. a double-slash) is disallowed
1109         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1110                              "400 Bad Request",
1111                              "The webapi does not allow empty pathname components",
1112                              self.PUT, self.public_url + "/foo//new.txt", "")
1113         return d
1114
1115     def test_DELETE_FILEURL(self):
1116         d = self.DELETE(self.public_url + "/foo/bar.txt")
1117         d.addCallback(lambda res:
1118                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1119         return d
1120
1121     def test_DELETE_FILEURL_missing(self):
1122         d = self.DELETE(self.public_url + "/foo/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1124         return d
1125
1126     def test_DELETE_FILEURL_missing2(self):
1127         d = self.DELETE(self.public_url + "/missing/missing")
1128         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1129         return d
1130
1131     def failUnlessHasBarDotTxtMetadata(self, res):
1132         data = simplejson.loads(res)
1133         self.failUnless(isinstance(data, list))
1134         self.failUnlessIn("metadata", data[1])
1135         self.failUnlessIn("tahoe", data[1]["metadata"])
1136         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1137         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1138         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1139                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1140
1141     def test_GET_FILEURL_json(self):
1142         # twisted.web.http.parse_qs ignores any query args without an '=', so
1143         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1144         # instead. This may make it tricky to emulate the S3 interface
1145         # completely.
1146         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1147         def _check1(data):
1148             self.failUnlessIsBarJSON(data)
1149             self.failUnlessHasBarDotTxtMetadata(data)
1150             return
1151         d.addCallback(_check1)
1152         return d
1153
1154     def test_GET_FILEURL_json_mutable_type(self):
1155         # The JSON should include format, which says whether the
1156         # file is SDMF or MDMF
1157         d = self.PUT("/uri?format=mdmf",
1158                      self.NEWFILE_CONTENTS * 300000)
1159         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1160         def _got_json(json, version):
1161             data = simplejson.loads(json)
1162             assert "filenode" == data[0]
1163             data = data[1]
1164             assert isinstance(data, dict)
1165
1166             self.failUnlessIn("format", data)
1167             self.failUnlessEqual(data["format"], version)
1168
1169         d.addCallback(_got_json, "MDMF")
1170         # Now make an SDMF file and check that it is reported correctly.
1171         d.addCallback(lambda ignored:
1172             self.PUT("/uri?format=sdmf",
1173                       self.NEWFILE_CONTENTS * 300000))
1174         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1175         d.addCallback(_got_json, "SDMF")
1176         return d
1177
1178     def test_GET_FILEURL_json_mdmf(self):
1179         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1180         d.addCallback(self.failUnlessIsQuuxJSON)
1181         return d
1182
1183     def test_GET_FILEURL_json_missing(self):
1184         d = self.GET(self.public_url + "/foo/missing?json")
1185         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1186         return d
1187
1188     def test_GET_FILEURL_uri(self):
1189         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1190         def _check(res):
1191             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1192         d.addCallback(_check)
1193         d.addCallback(lambda res:
1194                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1195         def _check2(res):
1196             # for now, for files, uris and readonly-uris are the same
1197             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1198         d.addCallback(_check2)
1199         return d
1200
1201     def test_GET_FILEURL_badtype(self):
1202         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1203                                  "bad t=bogus",
1204                                  self.GET,
1205                                  self.public_url + "/foo/bar.txt?t=bogus")
1206         return d
1207
1208     def test_CSS_FILE(self):
1209         d = self.GET("/tahoe.css", followRedirect=True)
1210         def _check(res):
1211             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1212             self.failUnless(CSS_STYLE.search(res), res)
1213         d.addCallback(_check)
1214         return d
1215
1216     def test_GET_FILEURL_uri_missing(self):
1217         d = self.GET(self.public_url + "/foo/missing?t=uri")
1218         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1219         return d
1220
1221     def _check_upload_and_mkdir_forms(self, html):
1222         # We should have a form to create a file, with radio buttons that allow
1223         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1224         self.failUnlessIn('name="t" value="upload"', html)
1225         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1226         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1227         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1228
1229         # We should also have the ability to create a mutable directory, with
1230         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1231         # or MDMF directory.
1232         self.failUnlessIn('name="t" value="mkdir"', html)
1233         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1234         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1235
1236         self.failUnlessIn(FAVICON_MARKUP, html)
1237
1238     def test_GET_DIRECTORY_html(self):
1239         d = self.GET(self.public_url + "/foo", followRedirect=True)
1240         def _check(html):
1241             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1242             self._check_upload_and_mkdir_forms(html)
1243             self.failUnlessIn("quux", html)
1244         d.addCallback(_check)
1245         return d
1246
1247     def test_GET_root_html(self):
1248         d = self.GET("/")
1249         d.addCallback(self._check_upload_and_mkdir_forms)
1250         return d
1251
1252     def test_GET_DIRURL(self):
1253         # the addSlash means we get a redirect here
1254         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1255         ROOT = "../../.."
1256         d = self.GET(self.public_url + "/foo", followRedirect=True)
1257         def _check(res):
1258             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1259
1260             # the FILE reference points to a URI, but it should end in bar.txt
1261             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1262                        (ROOT, urllib.quote(self._bar_txt_uri)))
1263             get_bar = "".join([r'<td>FILE</td>',
1264                                r'\s+<td>',
1265                                r'<a href="%s">bar.txt</a>' % bar_url,
1266                                r'</td>',
1267                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1268                                ])
1269             self.failUnless(re.search(get_bar, res), res)
1270             for label in ['unlink', 'rename/move']:
1271                 for line in res.split("\n"):
1272                     # find the line that contains the relevant button for bar.txt
1273                     if ("form action" in line and
1274                         ('value="%s"' % (label,)) in line and
1275                         'value="bar.txt"' in line):
1276                         # the form target should use a relative URL
1277                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1278                         self.failUnlessIn('action="%s"' % foo_url, line)
1279                         # and the when_done= should too
1280                         #done_url = urllib.quote(???)
1281                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1282
1283                         # 'unlink' needs to use POST because it directly has a side effect
1284                         if label == 'unlink':
1285                             self.failUnlessIn('method="post"', line)
1286                         break
1287                 else:
1288                     self.fail("unable to find '%s bar.txt' line" % (label,))
1289
1290             # the DIR reference just points to a URI
1291             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1292             get_sub = ((r'<td>DIR</td>')
1293                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1294             self.failUnless(re.search(get_sub, res), res)
1295         d.addCallback(_check)
1296
1297         # look at a readonly directory
1298         d.addCallback(lambda res:
1299                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1300         def _check2(res):
1301             self.failUnlessIn("(read-only)", res)
1302             self.failIfIn("Upload a file", res)
1303         d.addCallback(_check2)
1304
1305         # and at a directory that contains a readonly directory
1306         d.addCallback(lambda res:
1307                       self.GET(self.public_url, followRedirect=True))
1308         def _check3(res):
1309             self.failUnless(re.search('<td>DIR-RO</td>'
1310                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1311         d.addCallback(_check3)
1312
1313         # and an empty directory
1314         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1315         def _check4(res):
1316             self.failUnlessIn("directory is empty", res)
1317             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1318             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1319         d.addCallback(_check4)
1320
1321         # and at a literal directory
1322         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1323         d.addCallback(lambda res:
1324                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1325         def _check5(res):
1326             self.failUnlessIn('(immutable)', res)
1327             self.failUnless(re.search('<td>FILE</td>'
1328                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1329         d.addCallback(_check5)
1330         return d
1331
1332     def test_GET_DIRURL_badtype(self):
1333         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1334                                  400, "Bad Request",
1335                                  "bad t=bogus",
1336                                  self.GET,
1337                                  self.public_url + "/foo?t=bogus")
1338         return d
1339
1340     def test_GET_DIRURL_json(self):
1341         d = self.GET(self.public_url + "/foo?t=json")
1342         d.addCallback(self.failUnlessIsFooJSON)
1343         return d
1344
1345     def test_GET_DIRURL_json_format(self):
1346         d = self.PUT(self.public_url + \
1347                      "/foo/sdmf.txt?format=sdmf",
1348                      self.NEWFILE_CONTENTS * 300000)
1349         d.addCallback(lambda ignored:
1350             self.PUT(self.public_url + \
1351                      "/foo/mdmf.txt?format=mdmf",
1352                      self.NEWFILE_CONTENTS * 300000))
1353         # Now we have an MDMF and SDMF file in the directory. If we GET
1354         # its JSON, we should see their encodings.
1355         d.addCallback(lambda ignored:
1356             self.GET(self.public_url + "/foo?t=json"))
1357         def _got_json(json):
1358             data = simplejson.loads(json)
1359             assert data[0] == "dirnode"
1360
1361             data = data[1]
1362             kids = data['children']
1363
1364             mdmf_data = kids['mdmf.txt'][1]
1365             self.failUnlessIn("format", mdmf_data)
1366             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1367
1368             sdmf_data = kids['sdmf.txt'][1]
1369             self.failUnlessIn("format", sdmf_data)
1370             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1371         d.addCallback(_got_json)
1372         return d
1373
1374
1375     def test_POST_DIRURL_manifest_no_ophandle(self):
1376         d = self.shouldFail2(error.Error,
1377                              "test_POST_DIRURL_manifest_no_ophandle",
1378                              "400 Bad Request",
1379                              "slow operation requires ophandle=",
1380                              self.POST, self.public_url, t="start-manifest")
1381         return d
1382
1383     def test_POST_DIRURL_manifest(self):
1384         d = defer.succeed(None)
1385         def getman(ignored, output):
1386             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1387                           followRedirect=True)
1388             d.addCallback(self.wait_for_operation, "125")
1389             d.addCallback(self.get_operation_results, "125", output)
1390             return d
1391         d.addCallback(getman, None)
1392         def _got_html(manifest):
1393             self.failUnlessIn("Manifest of SI=", manifest)
1394             self.failUnlessIn("<td>sub</td>", manifest)
1395             self.failUnlessIn(self._sub_uri, manifest)
1396             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1397             self.failUnlessIn(FAVICON_MARKUP, manifest)
1398         d.addCallback(_got_html)
1399
1400         # both t=status and unadorned GET should be identical
1401         d.addCallback(lambda res: self.GET("/operations/125"))
1402         d.addCallback(_got_html)
1403
1404         d.addCallback(getman, "html")
1405         d.addCallback(_got_html)
1406         d.addCallback(getman, "text")
1407         def _got_text(manifest):
1408             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1409             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1410         d.addCallback(_got_text)
1411         d.addCallback(getman, "JSON")
1412         def _got_json(res):
1413             data = res["manifest"]
1414             got = {}
1415             for (path_list, cap) in data:
1416                 got[tuple(path_list)] = cap
1417             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1418             self.failUnlessIn((u"sub", u"baz.txt"), got)
1419             self.failUnlessIn("finished", res)
1420             self.failUnlessIn("origin", res)
1421             self.failUnlessIn("storage-index", res)
1422             self.failUnlessIn("verifycaps", res)
1423             self.failUnlessIn("stats", res)
1424         d.addCallback(_got_json)
1425         return d
1426
1427     def test_POST_DIRURL_deepsize_no_ophandle(self):
1428         d = self.shouldFail2(error.Error,
1429                              "test_POST_DIRURL_deepsize_no_ophandle",
1430                              "400 Bad Request",
1431                              "slow operation requires ophandle=",
1432                              self.POST, self.public_url, t="start-deep-size")
1433         return d
1434
1435     def test_POST_DIRURL_deepsize(self):
1436         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1437                       followRedirect=True)
1438         d.addCallback(self.wait_for_operation, "126")
1439         d.addCallback(self.get_operation_results, "126", "json")
1440         def _got_json(data):
1441             self.failUnlessReallyEqual(data["finished"], True)
1442             size = data["size"]
1443             self.failUnless(size > 1000)
1444         d.addCallback(_got_json)
1445         d.addCallback(self.get_operation_results, "126", "text")
1446         def _got_text(res):
1447             mo = re.search(r'^size: (\d+)$', res, re.M)
1448             self.failUnless(mo, res)
1449             size = int(mo.group(1))
1450             # with directories, the size varies.
1451             self.failUnless(size > 1000)
1452         d.addCallback(_got_text)
1453         return d
1454
1455     def test_POST_DIRURL_deepstats_no_ophandle(self):
1456         d = self.shouldFail2(error.Error,
1457                              "test_POST_DIRURL_deepstats_no_ophandle",
1458                              "400 Bad Request",
1459                              "slow operation requires ophandle=",
1460                              self.POST, self.public_url, t="start-deep-stats")
1461         return d
1462
1463     def test_POST_DIRURL_deepstats(self):
1464         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1465                       followRedirect=True)
1466         d.addCallback(self.wait_for_operation, "127")
1467         d.addCallback(self.get_operation_results, "127", "json")
1468         def _got_json(stats):
1469             expected = {"count-immutable-files": 3,
1470                         "count-mutable-files": 2,
1471                         "count-literal-files": 0,
1472                         "count-files": 5,
1473                         "count-directories": 3,
1474                         "size-immutable-files": 57,
1475                         "size-literal-files": 0,
1476                         #"size-directories": 1912, # varies
1477                         #"largest-directory": 1590,
1478                         "largest-directory-children": 7,
1479                         "largest-immutable-file": 19,
1480                         }
1481             for k,v in expected.iteritems():
1482                 self.failUnlessReallyEqual(stats[k], v,
1483                                            "stats[%s] was %s, not %s" %
1484                                            (k, stats[k], v))
1485             self.failUnlessReallyEqual(stats["size-files-histogram"],
1486                                        [ [11, 31, 3] ])
1487         d.addCallback(_got_json)
1488         return d
1489
1490     def test_POST_DIRURL_stream_manifest(self):
1491         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1492         def _check(res):
1493             self.failUnless(res.endswith("\n"))
1494             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1495             self.failUnlessReallyEqual(len(units), 9)
1496             self.failUnlessEqual(units[-1]["type"], "stats")
1497             first = units[0]
1498             self.failUnlessEqual(first["path"], [])
1499             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1500             self.failUnlessEqual(first["type"], "directory")
1501             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1502             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1503             self.failIfEqual(baz["storage-index"], None)
1504             self.failIfEqual(baz["verifycap"], None)
1505             self.failIfEqual(baz["repaircap"], None)
1506             # XXX: Add quux and baz to this test.
1507             return
1508         d.addCallback(_check)
1509         return d
1510
1511     def test_GET_DIRURL_uri(self):
1512         d = self.GET(self.public_url + "/foo?t=uri")
1513         def _check(res):
1514             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1515         d.addCallback(_check)
1516         return d
1517
1518     def test_GET_DIRURL_readonly_uri(self):
1519         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1520         def _check(res):
1521             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1522         d.addCallback(_check)
1523         return d
1524
1525     def test_PUT_NEWDIRURL(self):
1526         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1527         d.addCallback(lambda res:
1528                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1529         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1530         d.addCallback(self.failUnlessNodeKeysAre, [])
1531         return d
1532
1533     def test_PUT_NEWDIRURL_mdmf(self):
1534         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1535         d.addCallback(lambda res:
1536                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1537         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1538         d.addCallback(lambda node:
1539             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1540         return d
1541
1542     def test_PUT_NEWDIRURL_sdmf(self):
1543         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1544                      "")
1545         d.addCallback(lambda res:
1546                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1547         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1548         d.addCallback(lambda node:
1549             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1550         return d
1551
1552     def test_PUT_NEWDIRURL_bad_format(self):
1553         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1554                                     400, "Bad Request", "Unknown format: foo",
1555                                     self.PUT, self.public_url +
1556                                     "/foo/newdir=?t=mkdir&format=foo", "")
1557
1558     def test_POST_NEWDIRURL(self):
1559         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1560         d.addCallback(lambda res:
1561                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1562         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1563         d.addCallback(self.failUnlessNodeKeysAre, [])
1564         return d
1565
1566     def test_POST_NEWDIRURL_mdmf(self):
1567         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1568         d.addCallback(lambda res:
1569                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1570         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1571         d.addCallback(lambda node:
1572             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1573         return d
1574
1575     def test_POST_NEWDIRURL_sdmf(self):
1576         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1577         d.addCallback(lambda res:
1578             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1579         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1580         d.addCallback(lambda node:
1581             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1582         return d
1583
1584     def test_POST_NEWDIRURL_bad_format(self):
1585         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1586                                     400, "Bad Request", "Unknown format: foo",
1587                                     self.POST2, self.public_url + \
1588                                     "/foo/newdir?t=mkdir&format=foo", "")
1589
1590     def test_POST_NEWDIRURL_emptyname(self):
1591         # an empty pathname component (i.e. a double-slash) is disallowed
1592         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1593                              "400 Bad Request",
1594                              "The webapi does not allow empty pathname components, i.e. a double slash",
1595                              self.POST, self.public_url + "//?t=mkdir")
1596         return d
1597
1598     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1599         (newkids, caps) = self._create_initial_children()
1600         query = "/foo/newdir?t=mkdir-with-children"
1601         if version == MDMF_VERSION:
1602             query += "&format=mdmf"
1603         elif version == SDMF_VERSION:
1604             query += "&format=sdmf"
1605         else:
1606             version = SDMF_VERSION # for later
1607         d = self.POST2(self.public_url + query,
1608                        simplejson.dumps(newkids))
1609         def _check(uri):
1610             n = self.s.create_node_from_uri(uri.strip())
1611             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1612             self.failUnlessEqual(n._node.get_version(), version)
1613             d2.addCallback(lambda ign:
1614                            self.failUnlessROChildURIIs(n, u"child-imm",
1615                                                        caps['filecap1']))
1616             d2.addCallback(lambda ign:
1617                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1618                                                        caps['filecap2']))
1619             d2.addCallback(lambda ign:
1620                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1621                                                        caps['filecap3']))
1622             d2.addCallback(lambda ign:
1623                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1624                                                        caps['unknown_rocap']))
1625             d2.addCallback(lambda ign:
1626                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1627                                                        caps['unknown_rwcap']))
1628             d2.addCallback(lambda ign:
1629                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1630                                                        caps['unknown_immcap']))
1631             d2.addCallback(lambda ign:
1632                            self.failUnlessRWChildURIIs(n, u"dirchild",
1633                                                        caps['dircap']))
1634             d2.addCallback(lambda ign:
1635                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1636                                                        caps['litdircap']))
1637             d2.addCallback(lambda ign:
1638                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1639                                                        caps['emptydircap']))
1640             return d2
1641         d.addCallback(_check)
1642         d.addCallback(lambda res:
1643                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1644         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1645         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1648         return d
1649
1650     def test_POST_NEWDIRURL_initial_children(self):
1651         return self._do_POST_NEWDIRURL_initial_children_test()
1652
1653     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1654         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1655
1656     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1657         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1658
1659     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1660         (newkids, caps) = self._create_initial_children()
1661         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1662                                     400, "Bad Request", "Unknown format: foo",
1663                                     self.POST2, self.public_url + \
1664                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1665                                     simplejson.dumps(newkids))
1666
1667     def test_POST_NEWDIRURL_immutable(self):
1668         (newkids, caps) = self._create_immutable_children()
1669         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1670                        simplejson.dumps(newkids))
1671         def _check(uri):
1672             n = self.s.create_node_from_uri(uri.strip())
1673             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1674             d2.addCallback(lambda ign:
1675                            self.failUnlessROChildURIIs(n, u"child-imm",
1676                                                        caps['filecap1']))
1677             d2.addCallback(lambda ign:
1678                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1679                                                        caps['unknown_immcap']))
1680             d2.addCallback(lambda ign:
1681                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1682                                                        caps['immdircap']))
1683             d2.addCallback(lambda ign:
1684                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1685                                                        caps['litdircap']))
1686             d2.addCallback(lambda ign:
1687                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1688                                                        caps['emptydircap']))
1689             return d2
1690         d.addCallback(_check)
1691         d.addCallback(lambda res:
1692                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1693         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1694         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1696         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1700         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1702         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1704         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1705         d.addErrback(self.explain_web_error)
1706         return d
1707
1708     def test_POST_NEWDIRURL_immutable_bad(self):
1709         (newkids, caps) = self._create_initial_children()
1710         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1711                              "400 Bad Request",
1712                              "needed to be immutable but was not",
1713                              self.POST2,
1714                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1715                              simplejson.dumps(newkids))
1716         return d
1717
1718     def test_PUT_NEWDIRURL_exists(self):
1719         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1720         d.addCallback(lambda res:
1721                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1722         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1723         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1724         return d
1725
1726     def test_PUT_NEWDIRURL_blocked(self):
1727         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1728                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1729                              self.PUT,
1730                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1731         d.addCallback(lambda res:
1732                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1733         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1734         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1735         return d
1736
1737     def test_PUT_NEWDIRURL_mkdirs(self):
1738         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1739         d.addCallback(lambda res:
1740                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1741         d.addCallback(lambda res:
1742                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1743         d.addCallback(lambda res:
1744                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1745         d.addCallback(self.failUnlessNodeKeysAre, [])
1746         return d
1747
1748     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1749         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1750         d.addCallback(lambda ignored:
1751             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1752         d.addCallback(lambda ignored:
1753             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1754         d.addCallback(lambda ignored:
1755             self._foo_node.get_child_at_path(u"subdir"))
1756         def _got_subdir(subdir):
1757             # XXX: What we want?
1758             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1759             self.failUnlessNodeHasChild(subdir, u"newdir")
1760             return subdir.get_child_at_path(u"newdir")
1761         d.addCallback(_got_subdir)
1762         d.addCallback(lambda newdir:
1763             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1764         return d
1765
1766     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1767         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1768         d.addCallback(lambda ignored:
1769             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1770         d.addCallback(lambda ignored:
1771             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1772         d.addCallback(lambda ignored:
1773             self._foo_node.get_child_at_path(u"subdir"))
1774         def _got_subdir(subdir):
1775             # XXX: What we want?
1776             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1777             self.failUnlessNodeHasChild(subdir, u"newdir")
1778             return subdir.get_child_at_path(u"newdir")
1779         d.addCallback(_got_subdir)
1780         d.addCallback(lambda newdir:
1781             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1782         return d
1783
1784     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1785         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1786                                     400, "Bad Request", "Unknown format: foo",
1787                                     self.PUT, self.public_url + \
1788                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1789                                     "")
1790
1791     def test_DELETE_DIRURL(self):
1792         d = self.DELETE(self.public_url + "/foo")
1793         d.addCallback(lambda res:
1794                       self.failIfNodeHasChild(self.public_root, u"foo"))
1795         return d
1796
1797     def test_DELETE_DIRURL_missing(self):
1798         d = self.DELETE(self.public_url + "/foo/missing")
1799         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1800         d.addCallback(lambda res:
1801                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1802         return d
1803
1804     def test_DELETE_DIRURL_missing2(self):
1805         d = self.DELETE(self.public_url + "/missing")
1806         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1807         return d
1808
1809     def dump_root(self):
1810         print "NODEWALK"
1811         w = webish.DirnodeWalkerMixin()
1812         def visitor(childpath, childnode, metadata):
1813             print childpath
1814         d = w.walk(self.public_root, visitor)
1815         return d
1816
1817     def failUnlessNodeKeysAre(self, node, expected_keys):
1818         for k in expected_keys:
1819             assert isinstance(k, unicode)
1820         d = node.list()
1821         def _check(children):
1822             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1823         d.addCallback(_check)
1824         return d
1825     def failUnlessNodeHasChild(self, node, name):
1826         assert isinstance(name, unicode)
1827         d = node.list()
1828         def _check(children):
1829             self.failUnlessIn(name, children)
1830         d.addCallback(_check)
1831         return d
1832     def failIfNodeHasChild(self, node, name):
1833         assert isinstance(name, unicode)
1834         d = node.list()
1835         def _check(children):
1836             self.failIfIn(name, children)
1837         d.addCallback(_check)
1838         return d
1839
1840     def failUnlessChildContentsAre(self, node, name, expected_contents):
1841         assert isinstance(name, unicode)
1842         d = node.get_child_at_path(name)
1843         d.addCallback(lambda node: download_to_data(node))
1844         def _check(contents):
1845             self.failUnlessReallyEqual(contents, expected_contents)
1846         d.addCallback(_check)
1847         return d
1848
1849     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1850         assert isinstance(name, unicode)
1851         d = node.get_child_at_path(name)
1852         d.addCallback(lambda node: node.download_best_version())
1853         def _check(contents):
1854             self.failUnlessReallyEqual(contents, expected_contents)
1855         d.addCallback(_check)
1856         return d
1857
1858     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1859         assert isinstance(name, unicode)
1860         d = node.get_child_at_path(name)
1861         def _check(child):
1862             self.failUnless(child.is_unknown() or not child.is_readonly())
1863             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1864             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1865             expected_ro_uri = self._make_readonly(expected_uri)
1866             if expected_ro_uri:
1867                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1868         d.addCallback(_check)
1869         return d
1870
1871     def failUnlessROChildURIIs(self, node, name, expected_uri):
1872         assert isinstance(name, unicode)
1873         d = node.get_child_at_path(name)
1874         def _check(child):
1875             self.failUnless(child.is_unknown() or child.is_readonly())
1876             self.failUnlessReallyEqual(child.get_write_uri(), None)
1877             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1878             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1879         d.addCallback(_check)
1880         return d
1881
1882     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1883         assert isinstance(name, unicode)
1884         d = node.get_child_at_path(name)
1885         def _check(child):
1886             self.failUnless(child.is_unknown() or not child.is_readonly())
1887             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1888             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1889             expected_ro_uri = self._make_readonly(got_uri)
1890             if expected_ro_uri:
1891                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1892         d.addCallback(_check)
1893         return d
1894
1895     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1896         assert isinstance(name, unicode)
1897         d = node.get_child_at_path(name)
1898         def _check(child):
1899             self.failUnless(child.is_unknown() or child.is_readonly())
1900             self.failUnlessReallyEqual(child.get_write_uri(), None)
1901             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1902             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1903         d.addCallback(_check)
1904         return d
1905
1906     def failUnlessCHKURIHasContents(self, got_uri, contents):
1907         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1908
1909     def test_POST_upload(self):
1910         d = self.POST(self.public_url + "/foo", t="upload",
1911                       file=("new.txt", self.NEWFILE_CONTENTS))
1912         fn = self._foo_node
1913         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1914         d.addCallback(lambda res:
1915                       self.failUnlessChildContentsAre(fn, u"new.txt",
1916                                                       self.NEWFILE_CONTENTS))
1917         return d
1918
1919     def test_POST_upload_unicode(self):
1920         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1921         d = self.POST(self.public_url + "/foo", t="upload",
1922                       file=(filename, self.NEWFILE_CONTENTS))
1923         fn = self._foo_node
1924         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1925         d.addCallback(lambda res:
1926                       self.failUnlessChildContentsAre(fn, filename,
1927                                                       self.NEWFILE_CONTENTS))
1928         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1929         d.addCallback(lambda res: self.GET(target_url))
1930         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1931                                                                   self.NEWFILE_CONTENTS,
1932                                                                   contents))
1933         return d
1934
1935     def test_POST_upload_unicode_named(self):
1936         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1937         d = self.POST(self.public_url + "/foo", t="upload",
1938                       name=filename,
1939                       file=("overridden", self.NEWFILE_CONTENTS))
1940         fn = self._foo_node
1941         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1942         d.addCallback(lambda res:
1943                       self.failUnlessChildContentsAre(fn, filename,
1944                                                       self.NEWFILE_CONTENTS))
1945         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1946         d.addCallback(lambda res: self.GET(target_url))
1947         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1948                                                                   self.NEWFILE_CONTENTS,
1949                                                                   contents))
1950         return d
1951
1952     def test_POST_upload_no_link(self):
1953         d = self.POST("/uri", t="upload",
1954                       file=("new.txt", self.NEWFILE_CONTENTS))
1955         def _check_upload_results(page):
1956             # this should be a page which describes the results of the upload
1957             # that just finished.
1958             self.failUnlessIn("Upload Results:", page)
1959             self.failUnlessIn("URI:", page)
1960             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1961             mo = uri_re.search(page)
1962             self.failUnless(mo, page)
1963             new_uri = mo.group(1)
1964             return new_uri
1965         d.addCallback(_check_upload_results)
1966         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1967         return d
1968
1969     def test_POST_upload_no_link_whendone(self):
1970         d = self.POST("/uri", t="upload", when_done="/",
1971                       file=("new.txt", self.NEWFILE_CONTENTS))
1972         d.addBoth(self.shouldRedirect, "/")
1973         return d
1974
1975     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1976         d = defer.maybeDeferred(callable, *args, **kwargs)
1977         def done(res):
1978             if isinstance(res, failure.Failure):
1979                 res.trap(error.PageRedirect)
1980                 statuscode = res.value.status
1981                 target = res.value.location
1982                 return checker(statuscode, target)
1983             self.fail("%s: callable was supposed to redirect, not return '%s'"
1984                       % (which, res))
1985         d.addBoth(done)
1986         return d
1987
1988     def test_POST_upload_no_link_whendone_results(self):
1989         def check(statuscode, target):
1990             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1991             self.failUnless(target.startswith(self.webish_url), target)
1992             return client.getPage(target, method="GET")
1993         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1994                                  check,
1995                                  self.POST, "/uri", t="upload",
1996                                  when_done="/uri/%(uri)s",
1997                                  file=("new.txt", self.NEWFILE_CONTENTS))
1998         d.addCallback(lambda res:
1999                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2000         return d
2001
2002     def test_POST_upload_no_link_mutable(self):
2003         d = self.POST("/uri", t="upload", mutable="true",
2004                       file=("new.txt", self.NEWFILE_CONTENTS))
2005         def _check(filecap):
2006             filecap = filecap.strip()
2007             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2008             self.filecap = filecap
2009             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2010             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2011             n = self.s.create_node_from_uri(filecap)
2012             return n.download_best_version()
2013         d.addCallback(_check)
2014         def _check2(data):
2015             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2016             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2017         d.addCallback(_check2)
2018         def _check3(data):
2019             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2020             return self.GET("/file/%s" % urllib.quote(self.filecap))
2021         d.addCallback(_check3)
2022         def _check4(data):
2023             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2024         d.addCallback(_check4)
2025         return d
2026
2027     def test_POST_upload_no_link_mutable_toobig(self):
2028         # The SDMF size limit is no longer in place, so we should be
2029         # able to upload mutable files that are as large as we want them
2030         # to be.
2031         d = self.POST("/uri", t="upload", mutable="true",
2032                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2033         return d
2034
2035
2036     def test_POST_upload_format_unlinked(self):
2037         def _check_upload_unlinked(ign, format, uri_prefix):
2038             filename = format + ".txt"
2039             d = self.POST("/uri?t=upload&format=" + format,
2040                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2041             def _got_results(results):
2042                 if format.upper() in ("SDMF", "MDMF"):
2043                     # webapi.rst says this returns a filecap
2044                     filecap = results
2045                 else:
2046                     # for immutable, it returns an "upload results page", and
2047                     # the filecap is buried inside
2048                     line = [l for l in results.split("\n") if "URI: " in l][0]
2049                     mo = re.search(r'<span>([^<]+)</span>', line)
2050                     filecap = mo.group(1)
2051                 self.failUnless(filecap.startswith(uri_prefix),
2052                                 (uri_prefix, filecap))
2053                 return self.GET("/uri/%s?t=json" % filecap)
2054             d.addCallback(_got_results)
2055             def _got_json(json):
2056                 data = simplejson.loads(json)
2057                 data = data[1]
2058                 self.failUnlessIn("format", data)
2059                 self.failUnlessEqual(data["format"], format.upper())
2060             d.addCallback(_got_json)
2061             return d
2062         d = defer.succeed(None)
2063         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2064         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2065         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2066         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2067         return d
2068
2069     def test_POST_upload_bad_format_unlinked(self):
2070         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2071                                     400, "Bad Request", "Unknown format: foo",
2072                                     self.POST,
2073                                     "/uri?t=upload&format=foo",
2074                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2075
2076     def test_POST_upload_format(self):
2077         def _check_upload(ign, format, uri_prefix, fn=None):
2078             filename = format + ".txt"
2079             d = self.POST(self.public_url +
2080                           "/foo?t=upload&format=" + format,
2081                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2082             def _got_filecap(filecap):
2083                 if fn is not None:
2084                     filenameu = unicode(filename)
2085                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2086                 self.failUnless(filecap.startswith(uri_prefix))
2087                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2088             d.addCallback(_got_filecap)
2089             def _got_json(json):
2090                 data = simplejson.loads(json)
2091                 data = data[1]
2092                 self.failUnlessIn("format", data)
2093                 self.failUnlessEqual(data["format"], format.upper())
2094             d.addCallback(_got_json)
2095             return d
2096
2097         d = defer.succeed(None)
2098         d.addCallback(_check_upload, "chk", "URI:CHK")
2099         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2100         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2101         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2102         return d
2103
2104     def test_POST_upload_bad_format(self):
2105         return self.shouldHTTPError("POST_upload_bad_format",
2106                                     400, "Bad Request", "Unknown format: foo",
2107                                     self.POST, self.public_url + \
2108                                     "/foo?t=upload&format=foo",
2109                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2110
2111     def test_POST_upload_mutable(self):
2112         # this creates a mutable file
2113         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2114                       file=("new.txt", self.NEWFILE_CONTENTS))
2115         fn = self._foo_node
2116         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2117         d.addCallback(lambda res:
2118                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2119                                                              self.NEWFILE_CONTENTS))
2120         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2121         def _got(newnode):
2122             self.failUnless(IMutableFileNode.providedBy(newnode))
2123             self.failUnless(newnode.is_mutable())
2124             self.failIf(newnode.is_readonly())
2125             self._mutable_node = newnode
2126             self._mutable_uri = newnode.get_uri()
2127         d.addCallback(_got)
2128
2129         # now upload it again and make sure that the URI doesn't change
2130         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2131         d.addCallback(lambda res:
2132                       self.POST(self.public_url + "/foo", t="upload",
2133                                 mutable="true",
2134                                 file=("new.txt", NEWER_CONTENTS)))
2135         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2136         d.addCallback(lambda res:
2137                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2138                                                              NEWER_CONTENTS))
2139         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2140         def _got2(newnode):
2141             self.failUnless(IMutableFileNode.providedBy(newnode))
2142             self.failUnless(newnode.is_mutable())
2143             self.failIf(newnode.is_readonly())
2144             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2145         d.addCallback(_got2)
2146
2147         # upload a second time, using PUT instead of POST
2148         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2149         d.addCallback(lambda res:
2150                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2151         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2152         d.addCallback(lambda res:
2153                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2154                                                              NEW2_CONTENTS))
2155
2156         # finally list the directory, since mutable files are displayed
2157         # slightly differently
2158
2159         d.addCallback(lambda res:
2160                       self.GET(self.public_url + "/foo/",
2161                                followRedirect=True))
2162         def _check_page(res):
2163             # TODO: assert more about the contents
2164             self.failUnlessIn("SSK", res)
2165             return res
2166         d.addCallback(_check_page)
2167
2168         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2169         def _got3(newnode):
2170             self.failUnless(IMutableFileNode.providedBy(newnode))
2171             self.failUnless(newnode.is_mutable())
2172             self.failIf(newnode.is_readonly())
2173             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2174         d.addCallback(_got3)
2175
2176         # look at the JSON form of the enclosing directory
2177         d.addCallback(lambda res:
2178                       self.GET(self.public_url + "/foo/?t=json",
2179                                followRedirect=True))
2180         def _check_page_json(res):
2181             parsed = simplejson.loads(res)
2182             self.failUnlessEqual(parsed[0], "dirnode")
2183             children = dict( [(unicode(name),value)
2184                               for (name,value)
2185                               in parsed[1]["children"].iteritems()] )
2186             self.failUnlessIn(u"new.txt", children)
2187             new_json = children[u"new.txt"]
2188             self.failUnlessEqual(new_json[0], "filenode")
2189             self.failUnless(new_json[1]["mutable"])
2190             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2191             ro_uri = self._mutable_node.get_readonly().to_string()
2192             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2193         d.addCallback(_check_page_json)
2194
2195         # and the JSON form of the file
2196         d.addCallback(lambda res:
2197                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2198         def _check_file_json(res):
2199             parsed = simplejson.loads(res)
2200             self.failUnlessEqual(parsed[0], "filenode")
2201             self.failUnless(parsed[1]["mutable"])
2202             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2203             ro_uri = self._mutable_node.get_readonly().to_string()
2204             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2205         d.addCallback(_check_file_json)
2206
2207         # and look at t=uri and t=readonly-uri
2208         d.addCallback(lambda res:
2209                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2210         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2211         d.addCallback(lambda res:
2212                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2213         def _check_ro_uri(res):
2214             ro_uri = self._mutable_node.get_readonly().to_string()
2215             self.failUnlessReallyEqual(res, ro_uri)
2216         d.addCallback(_check_ro_uri)
2217
2218         # make sure we can get to it from /uri/URI
2219         d.addCallback(lambda res:
2220                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2221         d.addCallback(lambda res:
2222                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2223
2224         # and that HEAD computes the size correctly
2225         d.addCallback(lambda res:
2226                       self.HEAD(self.public_url + "/foo/new.txt",
2227                                 return_response=True))
2228         def _got_headers((res, status, headers)):
2229             self.failUnlessReallyEqual(res, "")
2230             self.failUnlessReallyEqual(headers["content-length"][0],
2231                                        str(len(NEW2_CONTENTS)))
2232             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2233         d.addCallback(_got_headers)
2234
2235         # make sure that outdated size limits aren't enforced anymore.
2236         d.addCallback(lambda ignored:
2237             self.POST(self.public_url + "/foo", t="upload",
2238                       mutable="true",
2239                       file=("new.txt",
2240                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2241         d.addErrback(self.dump_error)
2242         return d
2243
2244     def test_POST_upload_mutable_toobig(self):
2245         # SDMF had a size limti that was removed a while ago. MDMF has
2246         # never had a size limit. Test to make sure that we do not
2247         # encounter errors when trying to upload large mutable files,
2248         # since there should be no coded prohibitions regarding large
2249         # mutable files.
2250         d = self.POST(self.public_url + "/foo",
2251                       t="upload", mutable="true",
2252                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2253         return d
2254
2255     def dump_error(self, f):
2256         # if the web server returns an error code (like 400 Bad Request),
2257         # web.client.getPage puts the HTTP response body into the .response
2258         # attribute of the exception object that it gives back. It does not
2259         # appear in the Failure's repr(), so the ERROR that trial displays
2260         # will be rather terse and unhelpful. addErrback this method to the
2261         # end of your chain to get more information out of these errors.
2262         if f.check(error.Error):
2263             print "web.error.Error:"
2264             print f
2265             print f.value.response
2266         return f
2267
2268     def test_POST_upload_replace(self):
2269         d = self.POST(self.public_url + "/foo", t="upload",
2270                       file=("bar.txt", self.NEWFILE_CONTENTS))
2271         fn = self._foo_node
2272         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2273         d.addCallback(lambda res:
2274                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2275                                                       self.NEWFILE_CONTENTS))
2276         return d
2277
2278     def test_POST_upload_no_replace_ok(self):
2279         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2280                       file=("new.txt", self.NEWFILE_CONTENTS))
2281         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2282         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2283                                                              self.NEWFILE_CONTENTS))
2284         return d
2285
2286     def test_POST_upload_no_replace_queryarg(self):
2287         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2288                       file=("bar.txt", self.NEWFILE_CONTENTS))
2289         d.addBoth(self.shouldFail, error.Error,
2290                   "POST_upload_no_replace_queryarg",
2291                   "409 Conflict",
2292                   "There was already a child by that name, and you asked me "
2293                   "to not replace it")
2294         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2295         d.addCallback(self.failUnlessIsBarDotTxt)
2296         return d
2297
2298     def test_POST_upload_no_replace_field(self):
2299         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2300                       file=("bar.txt", self.NEWFILE_CONTENTS))
2301         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2302                   "409 Conflict",
2303                   "There was already a child by that name, and you asked me "
2304                   "to not replace it")
2305         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2306         d.addCallback(self.failUnlessIsBarDotTxt)
2307         return d
2308
2309     def test_POST_upload_whendone(self):
2310         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2311                       file=("new.txt", self.NEWFILE_CONTENTS))
2312         d.addBoth(self.shouldRedirect, "/THERE")
2313         fn = self._foo_node
2314         d.addCallback(lambda res:
2315                       self.failUnlessChildContentsAre(fn, u"new.txt",
2316                                                       self.NEWFILE_CONTENTS))
2317         return d
2318
2319     def test_POST_upload_named(self):
2320         fn = self._foo_node
2321         d = self.POST(self.public_url + "/foo", t="upload",
2322                       name="new.txt", file=self.NEWFILE_CONTENTS)
2323         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2324         d.addCallback(lambda res:
2325                       self.failUnlessChildContentsAre(fn, u"new.txt",
2326                                                       self.NEWFILE_CONTENTS))
2327         return d
2328
2329     def test_POST_upload_named_badfilename(self):
2330         d = self.POST(self.public_url + "/foo", t="upload",
2331                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2332         d.addBoth(self.shouldFail, error.Error,
2333                   "test_POST_upload_named_badfilename",
2334                   "400 Bad Request",
2335                   "name= may not contain a slash",
2336                   )
2337         # make sure that nothing was added
2338         d.addCallback(lambda res:
2339                       self.failUnlessNodeKeysAre(self._foo_node,
2340                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2341                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2342                                                   u"sub"]))
2343         return d
2344
2345     def test_POST_FILEURL_check(self):
2346         bar_url = self.public_url + "/foo/bar.txt"
2347         d = self.POST(bar_url, t="check")
2348         def _check(res):
2349             self.failUnlessIn("Healthy :", res)
2350         d.addCallback(_check)
2351         redir_url = "http://allmydata.org/TARGET"
2352         def _check2(statuscode, target):
2353             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2354             self.failUnlessReallyEqual(target, redir_url)
2355         d.addCallback(lambda res:
2356                       self.shouldRedirect2("test_POST_FILEURL_check",
2357                                            _check2,
2358                                            self.POST, bar_url,
2359                                            t="check",
2360                                            when_done=redir_url))
2361         d.addCallback(lambda res:
2362                       self.POST(bar_url, t="check", return_to=redir_url))
2363         def _check3(res):
2364             self.failUnlessIn("Healthy :", res)
2365             self.failUnlessIn("Return to file", res)
2366             self.failUnlessIn(redir_url, res)
2367         d.addCallback(_check3)
2368
2369         d.addCallback(lambda res:
2370                       self.POST(bar_url, t="check", output="JSON"))
2371         def _check_json(res):
2372             data = simplejson.loads(res)
2373             self.failUnlessIn("storage-index", data)
2374             self.failUnless(data["results"]["healthy"])
2375         d.addCallback(_check_json)
2376
2377         return d
2378
2379     def test_POST_FILEURL_check_and_repair(self):
2380         bar_url = self.public_url + "/foo/bar.txt"
2381         d = self.POST(bar_url, t="check", repair="true")
2382         def _check(res):
2383             self.failUnlessIn("Healthy :", res)
2384         d.addCallback(_check)
2385         redir_url = "http://allmydata.org/TARGET"
2386         def _check2(statuscode, target):
2387             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2388             self.failUnlessReallyEqual(target, redir_url)
2389         d.addCallback(lambda res:
2390                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2391                                            _check2,
2392                                            self.POST, bar_url,
2393                                            t="check", repair="true",
2394                                            when_done=redir_url))
2395         d.addCallback(lambda res:
2396                       self.POST(bar_url, t="check", return_to=redir_url))
2397         def _check3(res):
2398             self.failUnlessIn("Healthy :", res)
2399             self.failUnlessIn("Return to file", res)
2400             self.failUnlessIn(redir_url, res)
2401         d.addCallback(_check3)
2402         return d
2403
2404     def test_POST_DIRURL_check(self):
2405         foo_url = self.public_url + "/foo/"
2406         d = self.POST(foo_url, t="check")
2407         def _check(res):
2408             self.failUnlessIn("Healthy :", res)
2409         d.addCallback(_check)
2410         redir_url = "http://allmydata.org/TARGET"
2411         def _check2(statuscode, target):
2412             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2413             self.failUnlessReallyEqual(target, redir_url)
2414         d.addCallback(lambda res:
2415                       self.shouldRedirect2("test_POST_DIRURL_check",
2416                                            _check2,
2417                                            self.POST, foo_url,
2418                                            t="check",
2419                                            when_done=redir_url))
2420         d.addCallback(lambda res:
2421                       self.POST(foo_url, t="check", return_to=redir_url))
2422         def _check3(res):
2423             self.failUnlessIn("Healthy :", res)
2424             self.failUnlessIn("Return to file/directory", res)
2425             self.failUnlessIn(redir_url, res)
2426         d.addCallback(_check3)
2427
2428         d.addCallback(lambda res:
2429                       self.POST(foo_url, t="check", output="JSON"))
2430         def _check_json(res):
2431             data = simplejson.loads(res)
2432             self.failUnlessIn("storage-index", data)
2433             self.failUnless(data["results"]["healthy"])
2434         d.addCallback(_check_json)
2435
2436         return d
2437
2438     def test_POST_DIRURL_check_and_repair(self):
2439         foo_url = self.public_url + "/foo/"
2440         d = self.POST(foo_url, t="check", repair="true")
2441         def _check(res):
2442             self.failUnlessIn("Healthy :", res)
2443         d.addCallback(_check)
2444         redir_url = "http://allmydata.org/TARGET"
2445         def _check2(statuscode, target):
2446             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2447             self.failUnlessReallyEqual(target, redir_url)
2448         d.addCallback(lambda res:
2449                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2450                                            _check2,
2451                                            self.POST, foo_url,
2452                                            t="check", repair="true",
2453                                            when_done=redir_url))
2454         d.addCallback(lambda res:
2455                       self.POST(foo_url, t="check", return_to=redir_url))
2456         def _check3(res):
2457             self.failUnlessIn("Healthy :", res)
2458             self.failUnlessIn("Return to file/directory", res)
2459             self.failUnlessIn(redir_url, res)
2460         d.addCallback(_check3)
2461         return d
2462
2463     def test_POST_FILEURL_mdmf_check(self):
2464         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2465         d = self.POST(quux_url, t="check")
2466         def _check(res):
2467             self.failUnlessIn("Healthy", res)
2468         d.addCallback(_check)
2469         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2470         d.addCallback(lambda ignored:
2471                       self.POST(quux_extension_url, t="check"))
2472         d.addCallback(_check)
2473         return d
2474
2475     def test_POST_FILEURL_mdmf_check_and_repair(self):
2476         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2477         d = self.POST(quux_url, t="check", repair="true")
2478         def _check(res):
2479             self.failUnlessIn("Healthy", res)
2480         d.addCallback(_check)
2481         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2482         d.addCallback(lambda ignored:
2483                       self.POST(quux_extension_url, t="check", repair="true"))
2484         d.addCallback(_check)
2485         return d
2486
2487     def wait_for_operation(self, ignored, ophandle):
2488         url = "/operations/" + ophandle
2489         url += "?t=status&output=JSON"
2490         d = self.GET(url)
2491         def _got(res):
2492             data = simplejson.loads(res)
2493             if not data["finished"]:
2494                 d = self.stall(delay=1.0)
2495                 d.addCallback(self.wait_for_operation, ophandle)
2496                 return d
2497             return data
2498         d.addCallback(_got)
2499         return d
2500
2501     def get_operation_results(self, ignored, ophandle, output=None):
2502         url = "/operations/" + ophandle
2503         url += "?t=status"
2504         if output:
2505             url += "&output=" + output
2506         d = self.GET(url)
2507         def _got(res):
2508             if output and output.lower() == "json":
2509                 return simplejson.loads(res)
2510             return res
2511         d.addCallback(_got)
2512         return d
2513
2514     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2515         d = self.shouldFail2(error.Error,
2516                              "test_POST_DIRURL_deepcheck_no_ophandle",
2517                              "400 Bad Request",
2518                              "slow operation requires ophandle=",
2519                              self.POST, self.public_url, t="start-deep-check")
2520         return d
2521
2522     def test_POST_DIRURL_deepcheck(self):
2523         def _check_redirect(statuscode, target):
2524             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2525             self.failUnless(target.endswith("/operations/123"))
2526         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2527                                  self.POST, self.public_url,
2528                                  t="start-deep-check", ophandle="123")
2529         d.addCallback(self.wait_for_operation, "123")
2530         def _check_json(data):
2531             self.failUnlessReallyEqual(data["finished"], True)
2532             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2533             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2534         d.addCallback(_check_json)
2535         d.addCallback(self.get_operation_results, "123", "html")
2536         def _check_html(res):
2537             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2538             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2539             self.failUnlessIn(FAVICON_MARKUP, res)
2540         d.addCallback(_check_html)
2541
2542         d.addCallback(lambda res:
2543                       self.GET("/operations/123/"))
2544         d.addCallback(_check_html) # should be the same as without the slash
2545
2546         d.addCallback(lambda res:
2547                       self.shouldFail2(error.Error, "one", "404 Not Found",
2548                                        "No detailed results for SI bogus",
2549                                        self.GET, "/operations/123/bogus"))
2550
2551         foo_si = self._foo_node.get_storage_index()
2552         foo_si_s = base32.b2a(foo_si)
2553         d.addCallback(lambda res:
2554                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2555         def _check_foo_json(res):
2556             data = simplejson.loads(res)
2557             self.failUnlessEqual(data["storage-index"], foo_si_s)
2558             self.failUnless(data["results"]["healthy"])
2559         d.addCallback(_check_foo_json)
2560         return d
2561
2562     def test_POST_DIRURL_deepcheck_and_repair(self):
2563         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2564                       ophandle="124", output="json", followRedirect=True)
2565         d.addCallback(self.wait_for_operation, "124")
2566         def _check_json(data):
2567             self.failUnlessReallyEqual(data["finished"], True)
2568             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2569             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2570             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2571             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2572             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2573             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2574             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2575             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2576             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2577             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2578         d.addCallback(_check_json)
2579         d.addCallback(self.get_operation_results, "124", "html")
2580         def _check_html(res):
2581             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2582
2583             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2584             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2585             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2586
2587             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2588             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2589             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2590
2591             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2592             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2593             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2594
2595             self.failUnlessIn(FAVICON_MARKUP, res)
2596         d.addCallback(_check_html)
2597         return d
2598
2599     def test_POST_FILEURL_bad_t(self):
2600         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2601                              "POST to file: bad t=bogus",
2602                              self.POST, self.public_url + "/foo/bar.txt",
2603                              t="bogus")
2604         return d
2605
2606     def test_POST_mkdir(self): # return value?
2607         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2608         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2609         d.addCallback(self.failUnlessNodeKeysAre, [])
2610         return d
2611
2612     def test_POST_mkdir_mdmf(self):
2613         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2614         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2615         d.addCallback(lambda node:
2616             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2617         return d
2618
2619     def test_POST_mkdir_sdmf(self):
2620         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2621         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2622         d.addCallback(lambda node:
2623             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2624         return d
2625
2626     def test_POST_mkdir_bad_format(self):
2627         return self.shouldHTTPError("POST_mkdir_bad_format",
2628                                     400, "Bad Request", "Unknown format: foo",
2629                                     self.POST, self.public_url +
2630                                     "/foo?t=mkdir&name=newdir&format=foo")
2631
2632     def test_POST_mkdir_initial_children(self):
2633         (newkids, caps) = self._create_initial_children()
2634         d = self.POST2(self.public_url +
2635                        "/foo?t=mkdir-with-children&name=newdir",
2636                        simplejson.dumps(newkids))
2637         d.addCallback(lambda res:
2638                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2639         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2640         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2641         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2642         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2643         return d
2644
2645     def test_POST_mkdir_initial_children_mdmf(self):
2646         (newkids, caps) = self._create_initial_children()
2647         d = self.POST2(self.public_url +
2648                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2649                        simplejson.dumps(newkids))
2650         d.addCallback(lambda res:
2651                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2652         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2653         d.addCallback(lambda node:
2654             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2657                        caps['filecap1'])
2658         return d
2659
2660     # XXX: Duplication.
2661     def test_POST_mkdir_initial_children_sdmf(self):
2662         (newkids, caps) = self._create_initial_children()
2663         d = self.POST2(self.public_url +
2664                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2665                        simplejson.dumps(newkids))
2666         d.addCallback(lambda res:
2667                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2668         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2669         d.addCallback(lambda node:
2670             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2671         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2673                        caps['filecap1'])
2674         return d
2675
2676     def test_POST_mkdir_initial_children_bad_format(self):
2677         (newkids, caps) = self._create_initial_children()
2678         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2679                                     400, "Bad Request", "Unknown format: foo",
2680                                     self.POST, self.public_url + \
2681                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2682                                     simplejson.dumps(newkids))
2683
2684     def test_POST_mkdir_immutable(self):
2685         (newkids, caps) = self._create_immutable_children()
2686         d = self.POST2(self.public_url +
2687                        "/foo?t=mkdir-immutable&name=newdir",
2688                        simplejson.dumps(newkids))
2689         d.addCallback(lambda res:
2690                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2691         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2692         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2693         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2694         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2696         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2698         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2700         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2703         return d
2704
2705     def test_POST_mkdir_immutable_bad(self):
2706         (newkids, caps) = self._create_initial_children()
2707         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2708                              "400 Bad Request",
2709                              "needed to be immutable but was not",
2710                              self.POST2,
2711                              self.public_url +
2712                              "/foo?t=mkdir-immutable&name=newdir",
2713                              simplejson.dumps(newkids))
2714         return d
2715
2716     def test_POST_mkdir_2(self):
2717         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2718         d.addCallback(lambda res:
2719                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2720         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2721         d.addCallback(self.failUnlessNodeKeysAre, [])
2722         return d
2723
2724     def test_POST_mkdirs_2(self):
2725         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2726         d.addCallback(lambda res:
2727                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2728         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2729         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2730         d.addCallback(self.failUnlessNodeKeysAre, [])
2731         return d
2732
2733     def test_POST_mkdir_no_parentdir_noredirect(self):
2734         d = self.POST("/uri?t=mkdir")
2735         def _after_mkdir(res):
2736             uri.DirectoryURI.init_from_string(res)
2737         d.addCallback(_after_mkdir)
2738         return d
2739
2740     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2741         d = self.POST("/uri?t=mkdir&format=mdmf")
2742         def _after_mkdir(res):
2743             u = uri.from_string(res)
2744             # Check that this is an MDMF writecap
2745             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2746         d.addCallback(_after_mkdir)
2747         return d
2748
2749     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2750         d = self.POST("/uri?t=mkdir&format=sdmf")
2751         def _after_mkdir(res):
2752             u = uri.from_string(res)
2753             self.failUnlessIsInstance(u, uri.DirectoryURI)
2754         d.addCallback(_after_mkdir)
2755         return d
2756
2757     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2758         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2759                                     400, "Bad Request", "Unknown format: foo",
2760                                     self.POST, self.public_url +
2761                                     "/uri?t=mkdir&format=foo")
2762
2763     def test_POST_mkdir_no_parentdir_noredirect2(self):
2764         # make sure form-based arguments (as on the welcome page) still work
2765         d = self.POST("/uri", t="mkdir")
2766         def _after_mkdir(res):
2767             uri.DirectoryURI.init_from_string(res)
2768         d.addCallback(_after_mkdir)
2769         d.addErrback(self.explain_web_error)
2770         return d
2771
2772     def test_POST_mkdir_no_parentdir_redirect(self):
2773         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2774         d.addBoth(self.shouldRedirect, None, statuscode='303')
2775         def _check_target(target):
2776             target = urllib.unquote(target)
2777             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2778         d.addCallback(_check_target)
2779         return d
2780
2781     def test_POST_mkdir_no_parentdir_redirect2(self):
2782         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2783         d.addBoth(self.shouldRedirect, None, statuscode='303')
2784         def _check_target(target):
2785             target = urllib.unquote(target)
2786             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2787         d.addCallback(_check_target)
2788         d.addErrback(self.explain_web_error)
2789         return d
2790
2791     def _make_readonly(self, u):
2792         ro_uri = uri.from_string(u).get_readonly()
2793         if ro_uri is None:
2794             return None
2795         return ro_uri.to_string()
2796
2797     def _create_initial_children(self):
2798         contents, n, filecap1 = self.makefile(12)
2799         md1 = {"metakey1": "metavalue1"}
2800         filecap2 = make_mutable_file_uri()
2801         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2802         filecap3 = node3.get_readonly_uri()
2803         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2804         dircap = DirectoryNode(node4, None, None).get_uri()
2805         mdmfcap = make_mutable_file_uri(mdmf=True)
2806         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2807         emptydircap = "URI:DIR2-LIT:"
2808         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2809                                                       "ro_uri": self._make_readonly(filecap1),
2810                                                       "metadata": md1, }],
2811                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2812                                                       "ro_uri": self._make_readonly(filecap2)}],
2813                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2814                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2815                                                       "ro_uri": unknown_rocap}],
2816                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2817                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2818                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2819                                                       "ro_uri": self._make_readonly(dircap)}],
2820                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2821                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2822                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2823                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2824                    }
2825         return newkids, {'filecap1': filecap1,
2826                          'filecap2': filecap2,
2827                          'filecap3': filecap3,
2828                          'unknown_rwcap': unknown_rwcap,
2829                          'unknown_rocap': unknown_rocap,
2830                          'unknown_immcap': unknown_immcap,
2831                          'dircap': dircap,
2832                          'litdircap': litdircap,
2833                          'emptydircap': emptydircap,
2834                          'mdmfcap': mdmfcap}
2835
2836     def _create_immutable_children(self):
2837         contents, n, filecap1 = self.makefile(12)
2838         md1 = {"metakey1": "metavalue1"}
2839         tnode = create_chk_filenode("immutable directory contents\n"*10)
2840         dnode = DirectoryNode(tnode, None, None)
2841         assert not dnode.is_mutable()
2842         immdircap = dnode.get_uri()
2843         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2844         emptydircap = "URI:DIR2-LIT:"
2845         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2846                                                       "metadata": md1, }],
2847                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2848                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2849                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2850                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2851                    }
2852         return newkids, {'filecap1': filecap1,
2853                          'unknown_immcap': unknown_immcap,
2854                          'immdircap': immdircap,
2855                          'litdircap': litdircap,
2856                          'emptydircap': emptydircap}
2857
2858     def test_POST_mkdir_no_parentdir_initial_children(self):
2859         (newkids, caps) = self._create_initial_children()
2860         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2861         def _after_mkdir(res):
2862             self.failUnless(res.startswith("URI:DIR"), res)
2863             n = self.s.create_node_from_uri(res)
2864             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2865             d2.addCallback(lambda ign:
2866                            self.failUnlessROChildURIIs(n, u"child-imm",
2867                                                        caps['filecap1']))
2868             d2.addCallback(lambda ign:
2869                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2870                                                        caps['filecap2']))
2871             d2.addCallback(lambda ign:
2872                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2873                                                        caps['filecap3']))
2874             d2.addCallback(lambda ign:
2875                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2876                                                        caps['unknown_rwcap']))
2877             d2.addCallback(lambda ign:
2878                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2879                                                        caps['unknown_rocap']))
2880             d2.addCallback(lambda ign:
2881                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2882                                                        caps['unknown_immcap']))
2883             d2.addCallback(lambda ign:
2884                            self.failUnlessRWChildURIIs(n, u"dirchild",
2885                                                        caps['dircap']))
2886             return d2
2887         d.addCallback(_after_mkdir)
2888         return d
2889
2890     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2891         # the regular /uri?t=mkdir operation is specified to ignore its body.
2892         # Only t=mkdir-with-children pays attention to it.
2893         (newkids, caps) = self._create_initial_children()
2894         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2895                                  400, "Bad Request",
2896                                  "t=mkdir does not accept children=, "
2897                                  "try t=mkdir-with-children instead",
2898                                  self.POST2, "/uri?t=mkdir", # without children
2899                                  simplejson.dumps(newkids))
2900         return d
2901
2902     def test_POST_noparent_bad(self):
2903         d = self.shouldHTTPError("POST_noparent_bad",
2904                                  400, "Bad Request",
2905                                  "/uri accepts only PUT, PUT?t=mkdir, "
2906                                  "POST?t=upload, and POST?t=mkdir",
2907                                  self.POST, "/uri?t=bogus")
2908         return d
2909
2910     def test_POST_mkdir_no_parentdir_immutable(self):
2911         (newkids, caps) = self._create_immutable_children()
2912         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2913         def _after_mkdir(res):
2914             self.failUnless(res.startswith("URI:DIR"), res)
2915             n = self.s.create_node_from_uri(res)
2916             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2917             d2.addCallback(lambda ign:
2918                            self.failUnlessROChildURIIs(n, u"child-imm",
2919                                                           caps['filecap1']))
2920             d2.addCallback(lambda ign:
2921                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2922                                                           caps['unknown_immcap']))
2923             d2.addCallback(lambda ign:
2924                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2925                                                           caps['immdircap']))
2926             d2.addCallback(lambda ign:
2927                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2928                                                           caps['litdircap']))
2929             d2.addCallback(lambda ign:
2930                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2931                                                           caps['emptydircap']))
2932             return d2
2933         d.addCallback(_after_mkdir)
2934         return d
2935
2936     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2937         (newkids, caps) = self._create_initial_children()
2938         d = self.shouldFail2(error.Error,
2939                              "test_POST_mkdir_no_parentdir_immutable_bad",
2940                              "400 Bad Request",
2941                              "needed to be immutable but was not",
2942                              self.POST2,
2943                              "/uri?t=mkdir-immutable",
2944                              simplejson.dumps(newkids))
2945         return d
2946
2947     def test_welcome_page_mkdir_button(self):
2948         # Fetch the welcome page.
2949         d = self.GET("/")
2950         def _after_get_welcome_page(res):
2951             MKDIR_BUTTON_RE = re.compile(
2952                 '<form action="([^"]*)" method="post".*?'
2953                 '<input type="hidden" name="t" value="([^"]*)" />'
2954                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2955                 '<input type="submit" value="Create a directory" />',
2956                 re.I)
2957             mo = MKDIR_BUTTON_RE.search(res)
2958             formaction = mo.group(1)
2959             formt = mo.group(2)
2960             formaname = mo.group(3)
2961             formavalue = mo.group(4)
2962             return (formaction, formt, formaname, formavalue)
2963         d.addCallback(_after_get_welcome_page)
2964         def _after_parse_form(res):
2965             (formaction, formt, formaname, formavalue) = res
2966             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2967         d.addCallback(_after_parse_form)
2968         d.addBoth(self.shouldRedirect, None, statuscode='303')
2969         return d
2970
2971     def test_POST_mkdir_replace(self): # return value?
2972         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2973         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2974         d.addCallback(self.failUnlessNodeKeysAre, [])
2975         return d
2976
2977     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2978         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2979         d.addBoth(self.shouldFail, error.Error,
2980                   "POST_mkdir_no_replace_queryarg",
2981                   "409 Conflict",
2982                   "There was already a child by that name, and you asked me "
2983                   "to not replace it")
2984         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2985         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2986         return d
2987
2988     def test_POST_mkdir_no_replace_field(self): # return value?
2989         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2990                       replace="false")
2991         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2992                   "409 Conflict",
2993                   "There was already a child by that name, and you asked me "
2994                   "to not replace it")
2995         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2996         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2997         return d
2998
2999     def test_POST_mkdir_whendone_field(self):
3000         d = self.POST(self.public_url + "/foo",
3001                       t="mkdir", name="newdir", when_done="/THERE")
3002         d.addBoth(self.shouldRedirect, "/THERE")
3003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004         d.addCallback(self.failUnlessNodeKeysAre, [])
3005         return d
3006
3007     def test_POST_mkdir_whendone_queryarg(self):
3008         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3009                       t="mkdir", name="newdir")
3010         d.addBoth(self.shouldRedirect, "/THERE")
3011         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3012         d.addCallback(self.failUnlessNodeKeysAre, [])
3013         return d
3014
3015     def test_POST_bad_t(self):
3016         d = self.shouldFail2(error.Error, "POST_bad_t",
3017                              "400 Bad Request",
3018                              "POST to a directory with bad t=BOGUS",
3019                              self.POST, self.public_url + "/foo", t="BOGUS")
3020         return d
3021
3022     def test_POST_set_children(self, command_name="set_children"):
3023         contents9, n9, newuri9 = self.makefile(9)
3024         contents10, n10, newuri10 = self.makefile(10)
3025         contents11, n11, newuri11 = self.makefile(11)
3026
3027         reqbody = """{
3028                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3029                                                 "size": 0,
3030                                                 "metadata": {
3031                                                   "ctime": 1002777696.7564139,
3032                                                   "mtime": 1002777696.7564139
3033                                                  }
3034                                                } ],
3035                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3036                                                 "size": 1,
3037                                                 "metadata": {
3038                                                   "ctime": 1002777696.7564139,
3039                                                   "mtime": 1002777696.7564139
3040                                                  }
3041                                                } ],
3042                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3043                                                 "size": 2,
3044                                                 "metadata": {
3045                                                   "ctime": 1002777696.7564139,
3046                                                   "mtime": 1002777696.7564139
3047                                                  }
3048                                                } ]
3049                     }""" % (newuri9, newuri10, newuri11)
3050
3051         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3052
3053         d = client.getPage(url, method="POST", postdata=reqbody)
3054         def _then(res):
3055             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3056             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3057             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3058
3059         d.addCallback(_then)
3060         d.addErrback(self.dump_error)
3061         return d
3062
3063     def test_POST_set_children_with_hyphen(self):
3064         return self.test_POST_set_children(command_name="set-children")
3065
3066     def test_POST_link_uri(self):
3067         contents, n, newuri = self.makefile(8)
3068         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3069         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3070         d.addCallback(lambda res:
3071                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3072                                                       contents))
3073         return d
3074
3075     def test_POST_link_uri_replace(self):
3076         contents, n, newuri = self.makefile(8)
3077         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3078         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3079         d.addCallback(lambda res:
3080                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3081                                                       contents))
3082         return d
3083
3084     def test_POST_link_uri_unknown_bad(self):
3085         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3086         d.addBoth(self.shouldFail, error.Error,
3087                   "POST_link_uri_unknown_bad",
3088                   "400 Bad Request",
3089                   "unknown cap in a write slot")
3090         return d
3091
3092     def test_POST_link_uri_unknown_ro_good(self):
3093         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3094         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3095         return d
3096
3097     def test_POST_link_uri_unknown_imm_good(self):
3098         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3099         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3100         return d
3101
3102     def test_POST_link_uri_no_replace_queryarg(self):
3103         contents, n, newuri = self.makefile(8)
3104         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3105                       name="bar.txt", uri=newuri)
3106         d.addBoth(self.shouldFail, error.Error,
3107                   "POST_link_uri_no_replace_queryarg",
3108                   "409 Conflict",
3109                   "There was already a child by that name, and you asked me "
3110                   "to not replace it")
3111         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3112         d.addCallback(self.failUnlessIsBarDotTxt)
3113         return d
3114
3115     def test_POST_link_uri_no_replace_field(self):
3116         contents, n, newuri = self.makefile(8)
3117         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3118                       name="bar.txt", uri=newuri)
3119         d.addBoth(self.shouldFail, error.Error,
3120                   "POST_link_uri_no_replace_field",
3121                   "409 Conflict",
3122                   "There was already a child by that name, and you asked me "
3123                   "to not replace it")
3124         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3125         d.addCallback(self.failUnlessIsBarDotTxt)
3126         return d
3127
3128     def test_POST_delete(self, command_name='delete'):
3129         d = self._foo_node.list()
3130         def _check_before(children):
3131             self.failUnlessIn(u"bar.txt", children)
3132         d.addCallback(_check_before)
3133         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3134         d.addCallback(lambda res: self._foo_node.list())
3135         def _check_after(children):
3136             self.failIfIn(u"bar.txt", children)
3137         d.addCallback(_check_after)
3138         return d
3139
3140     def test_POST_unlink(self):
3141         return self.test_POST_delete(command_name='unlink')
3142
3143     def test_POST_rename_file(self):
3144         d = self.POST(self.public_url + "/foo", t="rename",
3145                       from_name="bar.txt", to_name='wibble.txt')
3146         d.addCallback(lambda res:
3147                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3148         d.addCallback(lambda res:
3149                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3150         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3151         d.addCallback(self.failUnlessIsBarDotTxt)
3152         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3153         d.addCallback(self.failUnlessIsBarJSON)
3154         return d
3155
3156     def test_POST_rename_file_redundant(self):
3157         d = self.POST(self.public_url + "/foo", t="rename",
3158                       from_name="bar.txt", to_name='bar.txt')
3159         d.addCallback(lambda res:
3160                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3161         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3162         d.addCallback(self.failUnlessIsBarDotTxt)
3163         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3164         d.addCallback(self.failUnlessIsBarJSON)
3165         return d
3166
3167     def test_POST_rename_file_replace(self):
3168         # rename a file and replace a directory with it
3169         d = self.POST(self.public_url + "/foo", t="rename",
3170                       from_name="bar.txt", to_name='empty')
3171         d.addCallback(lambda res:
3172                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3173         d.addCallback(lambda res:
3174                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3175         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3176         d.addCallback(self.failUnlessIsBarDotTxt)
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3178         d.addCallback(self.failUnlessIsBarJSON)
3179         return d
3180
3181     def test_POST_rename_file_no_replace_queryarg(self):
3182         # rename a file and replace a directory with it
3183         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3184                       from_name="bar.txt", to_name='empty')
3185         d.addBoth(self.shouldFail, error.Error,
3186                   "POST_rename_file_no_replace_queryarg",
3187                   "409 Conflict",
3188                   "There was already a child by that name, and you asked me "
3189                   "to not replace it")
3190         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3191         d.addCallback(self.failUnlessIsEmptyJSON)
3192         return d
3193
3194     def test_POST_rename_file_no_replace_field(self):
3195         # rename a file and replace a directory with it
3196         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3197                       from_name="bar.txt", to_name='empty')
3198         d.addBoth(self.shouldFail, error.Error,
3199                   "POST_rename_file_no_replace_field",
3200                   "409 Conflict",
3201                   "There was already a child by that name, and you asked me "
3202                   "to not replace it")
3203         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3204         d.addCallback(self.failUnlessIsEmptyJSON)
3205         return d
3206
3207     def failUnlessIsEmptyJSON(self, res):
3208         data = simplejson.loads(res)
3209         self.failUnlessEqual(data[0], "dirnode", data)
3210         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3211
3212     def test_POST_rename_file_slash_fail(self):
3213         d = self.POST(self.public_url + "/foo", t="rename",
3214                       from_name="bar.txt", to_name='kirk/spock.txt')
3215         d.addBoth(self.shouldFail, error.Error,
3216                   "test_POST_rename_file_slash_fail",
3217                   "400 Bad Request",
3218                   "to_name= may not contain a slash",
3219                   )
3220         d.addCallback(lambda res:
3221                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3222         return d
3223
3224     def test_POST_rename_dir(self):
3225         d = self.POST(self.public_url, t="rename",
3226                       from_name="foo", to_name='plunk')
3227         d.addCallback(lambda res:
3228                       self.failIfNodeHasChild(self.public_root, u"foo"))
3229         d.addCallback(lambda res:
3230                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3231         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3232         d.addCallback(self.failUnlessIsFooJSON)
3233         return d
3234
3235     def test_POST_move_file(self):
3236         d = self.POST(self.public_url + "/foo", t="move",
3237                       from_name="bar.txt", to_dir="sub")
3238         d.addCallback(lambda res:
3239                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3240         d.addCallback(lambda res:
3241                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3242         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3243         d.addCallback(self.failUnlessIsBarDotTxt)
3244         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3245         d.addCallback(self.failUnlessIsBarJSON)
3246         return d
3247
3248     def test_POST_move_file_new_name(self):
3249         d = self.POST(self.public_url + "/foo", t="move",
3250                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3251         d.addCallback(lambda res:
3252                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3253         d.addCallback(lambda res:
3254                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3255         d.addCallback(lambda res:
3256                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3257         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3258         d.addCallback(self.failUnlessIsBarDotTxt)
3259         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3260         d.addCallback(self.failUnlessIsBarJSON)
3261         return d
3262
3263     def test_POST_move_file_replace(self):
3264         d = self.POST(self.public_url + "/foo", t="move",
3265                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3266         d.addCallback(lambda res:
3267                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3268         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3269         d.addCallback(self.failUnlessIsBarDotTxt)
3270         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3271         d.addCallback(self.failUnlessIsBarJSON)
3272         return d
3273
3274     def test_POST_move_file_no_replace(self):
3275         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3276                              "409 Conflict",
3277                              "There was already a child by that name, and you asked me to not replace it",
3278                              self.POST, self.public_url + "/foo", t="move",
3279                              replace="false", from_name="bar.txt",
3280                              to_name="baz.txt", to_dir="sub")
3281         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3282         d.addCallback(self.failUnlessIsBarDotTxt)
3283         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3284         d.addCallback(self.failUnlessIsBarJSON)
3285         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3286         d.addCallback(self.failUnlessIsSubBazDotTxt)
3287         return d
3288
3289     def test_POST_move_file_slash_fail(self):
3290         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3291                              "400 Bad Request",
3292                              "to_name= may not contain a slash",
3293                              self.POST, self.public_url + "/foo", t="move",
3294                              from_name="bar.txt",
3295                              to_name="slash/fail.txt", to_dir="sub")
3296         d.addCallback(lambda res:
3297                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3298         d.addCallback(lambda res:
3299                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3300         d.addCallback(lambda ign:
3301                       self.shouldFail2(error.Error,
3302                                        "test_POST_rename_file_slash_fail2",
3303                                        "400 Bad Request",
3304                                        "from_name= may not contain a slash",
3305                                        self.POST, self.public_url + "/foo",
3306                                        t="move",
3307                                        from_name="nope/bar.txt",
3308                                        to_name="fail.txt", to_dir="sub"))
3309         return d
3310
3311     def test_POST_move_file_no_target(self):
3312         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3313                              "400 Bad Request",
3314                              "move requires from_name and to_dir",
3315                              self.POST, self.public_url + "/foo", t="move",
3316                              from_name="bar.txt", to_name="baz.txt")
3317         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3318         d.addCallback(self.failUnlessIsBarDotTxt)
3319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3320         d.addCallback(self.failUnlessIsBarJSON)
3321         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3322         d.addCallback(self.failUnlessIsBazDotTxt)
3323         return d
3324
3325     def test_POST_move_file_bad_target_type(self):
3326         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3327                              "400 Bad Request", "invalid target_type parameter",
3328                              self.POST,
3329                              self.public_url + "/foo", t="move",
3330                              target_type="*D", from_name="bar.txt",
3331                              to_dir="sub")
3332         return d
3333
3334     def test_POST_move_file_multi_level(self):
3335         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3336         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3337                       from_name="bar.txt", to_dir="sub/level2"))
3338         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3339         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3340         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3341         d.addCallback(self.failUnlessIsBarDotTxt)
3342         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3343         d.addCallback(self.failUnlessIsBarJSON)
3344         return d
3345
3346     def test_POST_move_file_to_uri(self):
3347         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3348                       from_name="bar.txt", to_dir=self._sub_uri)
3349         d.addCallback(lambda res:
3350                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3351         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3352         d.addCallback(self.failUnlessIsBarDotTxt)
3353         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3354         d.addCallback(self.failUnlessIsBarJSON)
3355         return d
3356
3357     def test_POST_move_file_to_nonexist_dir(self):
3358         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3359                             "404 Not Found", "No such child: nopechucktesta",
3360                             self.POST, self.public_url + "/foo", t="move",
3361                             from_name="bar.txt", to_dir="nopechucktesta")
3362         return d
3363
3364     def test_POST_move_file_into_file(self):
3365         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3366                              "400 Bad Request", "to_dir is not a directory",
3367                              self.POST, self.public_url + "/foo", t="move",
3368                              from_name="bar.txt", to_dir="baz.txt")
3369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3370         d.addCallback(self.failUnlessIsBazDotTxt)
3371         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3372         d.addCallback(self.failUnlessIsBarDotTxt)
3373         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3374         d.addCallback(self.failUnlessIsBarJSON)
3375         return d
3376
3377     def test_POST_move_file_to_bad_uri(self):
3378         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3379                               "400 Bad Request", "to_dir is not a directory",
3380                               self.POST, self.public_url + "/foo", t="move",
3381                               from_name="bar.txt", target_type="uri",
3382                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3383         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3384         d.addCallback(self.failUnlessIsBarDotTxt)
3385         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3386         d.addCallback(self.failUnlessIsBarJSON)
3387         return d
3388
3389     def test_POST_move_dir(self):
3390         d = self.POST(self.public_url + "/foo", t="move",
3391                       from_name="bar.txt", to_dir="empty")
3392         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3393                       t="move", from_name="empty", to_dir="sub"))
3394         d.addCallback(lambda res:
3395                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3396         d.addCallback(lambda res:
3397                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3398         d.addCallback(lambda res:
3399                       self._sub_node.get_child_at_path(u"empty"))
3400         d.addCallback(lambda node:
3401                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3402         d.addCallback(lambda res:
3403                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3404         d.addCallback(self.failUnlessIsBarDotTxt)
3405         return d
3406
3407     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3408         """ If target is not None then the redirection has to go to target.  If
3409         statuscode is not None then the redirection has to be accomplished with
3410         that HTTP status code."""
3411         if not isinstance(res, failure.Failure):
3412             to_where = (target is None) and "somewhere" or ("to " + target)
3413             self.fail("%s: we were expecting to get redirected %s, not get an"
3414                       " actual page: %s" % (which, to_where, res))
3415         res.trap(error.PageRedirect)
3416         if statuscode is not None:
3417             self.failUnlessReallyEqual(res.value.status, statuscode,
3418                                        "%s: not a redirect" % which)
3419         if target is not None:
3420             # the PageRedirect does not seem to capture the uri= query arg
3421             # properly, so we can't check for it.
3422             realtarget = self.webish_url + target
3423             self.failUnlessReallyEqual(res.value.location, realtarget,
3424                                        "%s: wrong target" % which)
3425         return res.value.location
3426
3427     def test_GET_URI_form(self):
3428         base = "/uri?uri=%s" % self._bar_txt_uri
3429         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3430         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3431         d = self.GET(base)
3432         d.addBoth(self.shouldRedirect, targetbase)
3433         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3434         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3435         d.addCallback(lambda res: self.GET(base+"&t=json"))
3436         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3437         d.addCallback(self.log, "about to get file by uri")
3438         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3439         d.addCallback(self.failUnlessIsBarDotTxt)
3440         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3441         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3442                                            followRedirect=True))
3443         d.addCallback(self.failUnlessIsFooJSON)
3444         d.addCallback(self.log, "got dir by uri")
3445
3446         return d
3447
3448     def test_GET_URI_form_bad(self):
3449         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3450                              "400 Bad Request", "GET /uri requires uri=",
3451                              self.GET, "/uri")
3452         return d
3453
3454     def test_GET_rename_form(self):
3455         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3456                      followRedirect=True)
3457         def _check(res):
3458             self.failUnlessIn('name="when_done" value="."', res)
3459             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3460             self.failUnlessIn(FAVICON_MARKUP, res)
3461         d.addCallback(_check)
3462         return d
3463
3464     def log(self, res, msg):
3465         #print "MSG: %s  RES: %s" % (msg, res)
3466         log.msg(msg)
3467         return res
3468
3469     def test_GET_URI_URL(self):
3470         base = "/uri/%s" % self._bar_txt_uri
3471         d = self.GET(base)
3472         d.addCallback(self.failUnlessIsBarDotTxt)
3473         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3474         d.addCallback(self.failUnlessIsBarDotTxt)
3475         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3476         d.addCallback(self.failUnlessIsBarDotTxt)
3477         return d
3478
3479     def test_GET_URI_URL_dir(self):
3480         base = "/uri/%s?t=json" % self._foo_uri
3481         d = self.GET(base)
3482         d.addCallback(self.failUnlessIsFooJSON)
3483         return d
3484
3485     def test_GET_URI_URL_missing(self):
3486         base = "/uri/%s" % self._bad_file_uri
3487         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3488                                  http.GONE, None, "NotEnoughSharesError",
3489                                  self.GET, base)
3490         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3491         # here? we must arrange for a download to fail after target.open()
3492         # has been called, and then inspect the response to see that it is
3493         # shorter than we expected.
3494         return d
3495
3496     def test_PUT_DIRURL_uri(self):
3497         d = self.s.create_dirnode()
3498         def _made_dir(dn):
3499             new_uri = dn.get_uri()
3500             # replace /foo with a new (empty) directory
3501             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3502             d.addCallback(lambda res:
3503                           self.failUnlessReallyEqual(res.strip(), new_uri))
3504             d.addCallback(lambda res:
3505                           self.failUnlessRWChildURIIs(self.public_root,
3506                                                       u"foo",
3507                                                       new_uri))
3508             return d
3509         d.addCallback(_made_dir)
3510         return d
3511
3512     def test_PUT_DIRURL_uri_noreplace(self):
3513         d = self.s.create_dirnode()
3514         def _made_dir(dn):
3515             new_uri = dn.get_uri()
3516             # replace /foo with a new (empty) directory, but ask that
3517             # replace=false, so it should fail
3518             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3519                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3520                                  self.PUT,
3521                                  self.public_url + "/foo?t=uri&replace=false",
3522                                  new_uri)
3523             d.addCallback(lambda res:
3524                           self.failUnlessRWChildURIIs(self.public_root,
3525                                                       u"foo",
3526                                                       self._foo_uri))
3527             return d
3528         d.addCallback(_made_dir)
3529         return d
3530
3531     def test_PUT_DIRURL_bad_t(self):
3532         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3533                              "400 Bad Request", "PUT to a directory",
3534                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3535         d.addCallback(lambda res:
3536                       self.failUnlessRWChildURIIs(self.public_root,
3537                                                   u"foo",
3538                                                   self._foo_uri))
3539         return d
3540
3541     def test_PUT_NEWFILEURL_uri(self):
3542         contents, n, new_uri = self.makefile(8)
3543         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3544         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3545         d.addCallback(lambda res:
3546                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3547                                                       contents))
3548         return d
3549
3550     def test_PUT_NEWFILEURL_mdmf(self):
3551         new_contents = self.NEWFILE_CONTENTS * 300000
3552         d = self.PUT(self.public_url + \
3553                      "/foo/mdmf.txt?format=mdmf",
3554                      new_contents)
3555         d.addCallback(lambda ignored:
3556             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3557         def _got_json(json):
3558             data = simplejson.loads(json)
3559             data = data[1]
3560             self.failUnlessIn("format", data)
3561             self.failUnlessEqual(data["format"], "MDMF")
3562             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3563             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3564         d.addCallback(_got_json)
3565         return d
3566
3567     def test_PUT_NEWFILEURL_sdmf(self):
3568         new_contents = self.NEWFILE_CONTENTS * 300000
3569         d = self.PUT(self.public_url + \
3570                      "/foo/sdmf.txt?format=sdmf",
3571                      new_contents)
3572         d.addCallback(lambda ignored:
3573             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3574         def _got_json(json):
3575             data = simplejson.loads(json)
3576             data = data[1]
3577             self.failUnlessIn("format", data)
3578             self.failUnlessEqual(data["format"], "SDMF")
3579         d.addCallback(_got_json)
3580         return d
3581
3582     def test_PUT_NEWFILEURL_bad_format(self):
3583        new_contents = self.NEWFILE_CONTENTS * 300000
3584        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3585                                    400, "Bad Request", "Unknown format: foo",
3586                                    self.PUT, self.public_url + \
3587                                    "/foo/foo.txt?format=foo",
3588                                    new_contents)
3589
3590     def test_PUT_NEWFILEURL_uri_replace(self):
3591         contents, n, new_uri = self.makefile(8)
3592         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3593         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3594         d.addCallback(lambda res:
3595                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3596                                                       contents))
3597         return d
3598
3599     def test_PUT_NEWFILEURL_uri_no_replace(self):
3600         contents, n, new_uri = self.makefile(8)
3601         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3602         d.addBoth(self.shouldFail, error.Error,
3603                   "PUT_NEWFILEURL_uri_no_replace",
3604                   "409 Conflict",
3605                   "There was already a child by that name, and you asked me "
3606                   "to not replace it")
3607         return d
3608
3609     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3610         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3611         d.addBoth(self.shouldFail, error.Error,
3612                   "POST_put_uri_unknown_bad",
3613                   "400 Bad Request",
3614                   "unknown cap in a write slot")
3615         return d
3616
3617     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3618         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3619         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3620                       u"put-future-ro.txt")
3621         return d
3622
3623     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3624         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3625         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3626                       u"put-future-imm.txt")
3627         return d
3628
3629     def test_PUT_NEWFILE_URI(self):
3630         file_contents = "New file contents here\n"
3631         d = self.PUT("/uri", file_contents)
3632         def _check(uri):
3633             assert isinstance(uri, str), uri
3634             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3635             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3636                                        file_contents)
3637             return self.GET("/uri/%s" % uri)
3638         d.addCallback(_check)
3639         def _check2(res):
3640             self.failUnlessReallyEqual(res, file_contents)
3641         d.addCallback(_check2)
3642         return d
3643
3644     def test_PUT_NEWFILE_URI_not_mutable(self):
3645         file_contents = "New file contents here\n"
3646         d = self.PUT("/uri?mutable=false", file_contents)
3647         def _check(uri):
3648             assert isinstance(uri, str), uri
3649             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3650             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3651                                        file_contents)
3652             return self.GET("/uri/%s" % uri)
3653         d.addCallback(_check)
3654         def _check2(res):
3655             self.failUnlessReallyEqual(res, file_contents)
3656         d.addCallback(_check2)
3657         return d
3658
3659     def test_PUT_NEWFILE_URI_only_PUT(self):
3660         d = self.PUT("/uri?t=bogus", "")
3661         d.addBoth(self.shouldFail, error.Error,
3662                   "PUT_NEWFILE_URI_only_PUT",
3663                   "400 Bad Request",
3664                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3665         return d
3666
3667     def test_PUT_NEWFILE_URI_mutable(self):
3668         file_contents = "New file contents here\n"
3669         d = self.PUT("/uri?mutable=true", file_contents)
3670         def _check1(filecap):
3671             filecap = filecap.strip()
3672             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3673             self.filecap = filecap
3674             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3675             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3676             n = self.s.create_node_from_uri(filecap)
3677             return n.download_best_version()
3678         d.addCallback(_check1)
3679         def _check2(data):
3680             self.failUnlessReallyEqual(data, file_contents)
3681             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3682         d.addCallback(_check2)
3683         def _check3(res):
3684             self.failUnlessReallyEqual(res, file_contents)
3685         d.addCallback(_check3)
3686         return d
3687
3688     def test_PUT_mkdir(self):
3689         d = self.PUT("/uri?t=mkdir", "")
3690         def _check(uri):
3691             n = self.s.create_node_from_uri(uri.strip())
3692             d2 = self.failUnlessNodeKeysAre(n, [])
3693             d2.addCallback(lambda res:
3694                            self.GET("/uri/%s?t=json" % uri))
3695             return d2
3696         d.addCallback(_check)
3697         d.addCallback(self.failUnlessIsEmptyJSON)
3698         return d
3699
3700     def test_PUT_mkdir_mdmf(self):
3701         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3702         def _got(res):
3703             u = uri.from_string(res)
3704             # Check that this is an MDMF writecap
3705             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3706         d.addCallback(_got)
3707         return d
3708
3709     def test_PUT_mkdir_sdmf(self):
3710         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3711         def _got(res):
3712             u = uri.from_string(res)
3713             self.failUnlessIsInstance(u, uri.DirectoryURI)
3714         d.addCallback(_got)
3715         return d
3716
3717     def test_PUT_mkdir_bad_format(self):
3718         return self.shouldHTTPError("PUT_mkdir_bad_format",
3719                                     400, "Bad Request", "Unknown format: foo",
3720                                     self.PUT, "/uri?t=mkdir&format=foo",
3721                                     "")
3722
3723     def test_POST_check(self):
3724         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3725         def _done(res):
3726             # this returns a string form of the results, which are probably
3727             # None since we're using fake filenodes.
3728             # TODO: verify that the check actually happened, by changing
3729             # FakeCHKFileNode to count how many times .check() has been
3730             # called.
3731             pass
3732         d.addCallback(_done)
3733         return d
3734
3735
3736     def test_PUT_update_at_offset(self):
3737         file_contents = "test file" * 100000 # about 900 KiB
3738         d = self.PUT("/uri?mutable=true", file_contents)
3739         def _then(filecap):
3740             self.filecap = filecap
3741             new_data = file_contents[:100]
3742             new = "replaced and so on"
3743             new_data += new
3744             new_data += file_contents[len(new_data):]
3745             assert len(new_data) == len(file_contents)
3746             self.new_data = new_data
3747         d.addCallback(_then)
3748         d.addCallback(lambda ignored:
3749             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3750                      "replaced and so on"))
3751         def _get_data(filecap):
3752             n = self.s.create_node_from_uri(filecap)
3753             return n.download_best_version()
3754         d.addCallback(_get_data)
3755         d.addCallback(lambda results:
3756             self.failUnlessEqual(results, self.new_data))
3757         # Now try appending things to the file
3758         d.addCallback(lambda ignored:
3759             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3760                      "puppies" * 100))
3761         d.addCallback(_get_data)
3762         d.addCallback(lambda results:
3763             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3764         # and try replacing the beginning of the file
3765         d.addCallback(lambda ignored:
3766             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3767         d.addCallback(_get_data)
3768         d.addCallback(lambda results:
3769             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3770         return d
3771
3772     def test_PUT_update_at_invalid_offset(self):
3773         file_contents = "test file" * 100000 # about 900 KiB
3774         d = self.PUT("/uri?mutable=true", file_contents)
3775         def _then(filecap):
3776             self.filecap = filecap
3777         d.addCallback(_then)
3778         # Negative offsets should cause an error.
3779         d.addCallback(lambda ignored:
3780             self.shouldHTTPError("PUT_update_at_invalid_offset",
3781                                  400, "Bad Request",
3782                                  "Invalid offset",
3783                                  self.PUT,
3784                                  "/uri/%s?offset=-1" % self.filecap,
3785                                  "foo"))
3786         return d
3787
3788     def test_PUT_update_at_offset_immutable(self):
3789         file_contents = "Test file" * 100000
3790         d = self.PUT("/uri", file_contents)
3791         def _then(filecap):
3792             self.filecap = filecap
3793         d.addCallback(_then)
3794         d.addCallback(lambda ignored:
3795             self.shouldHTTPError("PUT_update_at_offset_immutable",
3796                                  400, "Bad Request",
3797                                  "immutable",
3798                                  self.PUT,
3799                                  "/uri/%s?offset=50" % self.filecap,
3800                                  "foo"))
3801         return d
3802
3803
3804     def test_bad_method(self):
3805         url = self.webish_url + self.public_url + "/foo/bar.txt"
3806         d = self.shouldHTTPError("bad_method",
3807                                  501, "Not Implemented",
3808                                  "I don't know how to treat a BOGUS request.",
3809                                  client.getPage, url, method="BOGUS")
3810         return d
3811
3812     def test_short_url(self):
3813         url = self.webish_url + "/uri"
3814         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3815                                  "I don't know how to treat a DELETE request.",
3816                                  client.getPage, url, method="DELETE")
3817         return d
3818
3819     def test_ophandle_bad(self):
3820         url = self.webish_url + "/operations/bogus?t=status"
3821         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3822                                  "unknown/expired handle 'bogus'",
3823                                  client.getPage, url)
3824         return d
3825
3826     def test_ophandle_cancel(self):
3827         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3828                       followRedirect=True)
3829         d.addCallback(lambda ignored:
3830                       self.GET("/operations/128?t=status&output=JSON"))
3831         def _check1(res):
3832             data = simplejson.loads(res)
3833             self.failUnless("finished" in data, res)
3834             monitor = self.ws.root.child_operations.handles["128"][0]
3835             d = self.POST("/operations/128?t=cancel&output=JSON")
3836             def _check2(res):
3837                 data = simplejson.loads(res)
3838                 self.failUnless("finished" in data, res)
3839                 # t=cancel causes the handle to be forgotten
3840                 self.failUnless(monitor.is_cancelled())
3841             d.addCallback(_check2)
3842             return d
3843         d.addCallback(_check1)
3844         d.addCallback(lambda ignored:
3845                       self.shouldHTTPError("ophandle_cancel",
3846                                            404, "404 Not Found",
3847                                            "unknown/expired handle '128'",
3848                                            self.GET,
3849                                            "/operations/128?t=status&output=JSON"))
3850         return d
3851
3852     def test_ophandle_retainfor(self):
3853         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3854                       followRedirect=True)
3855         d.addCallback(lambda ignored:
3856                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3857         def _check1(res):
3858             data = simplejson.loads(res)
3859             self.failUnless("finished" in data, res)
3860         d.addCallback(_check1)
3861         # the retain-for=0 will cause the handle to be expired very soon
3862         d.addCallback(lambda ign:
3863             self.clock.advance(2.0))
3864         d.addCallback(lambda ignored:
3865                       self.shouldHTTPError("ophandle_retainfor",
3866                                            404, "404 Not Found",
3867                                            "unknown/expired handle '129'",
3868                                            self.GET,
3869                                            "/operations/129?t=status&output=JSON"))
3870         return d
3871
3872     def test_ophandle_release_after_complete(self):
3873         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3874                       followRedirect=True)
3875         d.addCallback(self.wait_for_operation, "130")
3876         d.addCallback(lambda ignored:
3877                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3878         # the release-after-complete=true will cause the handle to be expired
3879         d.addCallback(lambda ignored:
3880                       self.shouldHTTPError("ophandle_release_after_complete",
3881                                            404, "404 Not Found",
3882                                            "unknown/expired handle '130'",
3883                                            self.GET,
3884                                            "/operations/130?t=status&output=JSON"))
3885         return d
3886
3887     def test_uncollected_ophandle_expiration(self):
3888         # uncollected ophandles should expire after 4 days
3889         def _make_uncollected_ophandle(ophandle):
3890             d = self.POST(self.public_url +
3891                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3892                           followRedirect=False)
3893             # When we start the operation, the webapi server will want
3894             # to redirect us to the page for the ophandle, so we get
3895             # confirmation that the operation has started. If the
3896             # manifest operation has finished by the time we get there,
3897             # following that redirect (by setting followRedirect=True
3898             # above) has the side effect of collecting the ophandle that
3899             # we've just created, which means that we can't use the
3900             # ophandle to test the uncollected timeout anymore. So,
3901             # instead, catch the 302 here and don't follow it.
3902             d.addBoth(self.should302, "uncollected_ophandle_creation")
3903             return d
3904         # Create an ophandle, don't collect it, then advance the clock by
3905         # 4 days - 1 second and make sure that the ophandle is still there.
3906         d = _make_uncollected_ophandle(131)
3907         d.addCallback(lambda ign:
3908             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3909         d.addCallback(lambda ign:
3910             self.GET("/operations/131?t=status&output=JSON"))
3911         def _check1(res):
3912             data = simplejson.loads(res)
3913             self.failUnless("finished" in data, res)
3914         d.addCallback(_check1)
3915         # Create an ophandle, don't collect it, then try to collect it
3916         # after 4 days. It should be gone.
3917         d.addCallback(lambda ign:
3918             _make_uncollected_ophandle(132))
3919         d.addCallback(lambda ign:
3920             self.clock.advance(96*60*60))
3921         d.addCallback(lambda ign:
3922             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3923                                  404, "404 Not Found",
3924                                  "unknown/expired handle '132'",
3925                                  self.GET,
3926                                  "/operations/132?t=status&output=JSON"))
3927         return d
3928
3929     def test_collected_ophandle_expiration(self):
3930         # collected ophandles should expire after 1 day
3931         def _make_collected_ophandle(ophandle):
3932             d = self.POST(self.public_url +
3933                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3934                           followRedirect=True)
3935             # By following the initial redirect, we collect the ophandle
3936             # we've just created.
3937             return d
3938         # Create a collected ophandle, then collect it after 23 hours
3939         # and 59 seconds to make sure that it is still there.
3940         d = _make_collected_ophandle(133)
3941         d.addCallback(lambda ign:
3942             self.clock.advance((24*60*60) - 1))
3943         d.addCallback(lambda ign:
3944             self.GET("/operations/133?t=status&output=JSON"))
3945         def _check1(res):
3946             data = simplejson.loads(res)
3947             self.failUnless("finished" in data, res)
3948         d.addCallback(_check1)
3949         # Create another uncollected ophandle, then try to collect it
3950         # after 24 hours to make sure that it is gone.
3951         d.addCallback(lambda ign:
3952             _make_collected_ophandle(134))
3953         d.addCallback(lambda ign:
3954             self.clock.advance(24*60*60))
3955         d.addCallback(lambda ign:
3956             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3957                                  404, "404 Not Found",
3958                                  "unknown/expired handle '134'",
3959                                  self.GET,
3960                                  "/operations/134?t=status&output=JSON"))
3961         return d
3962
3963     def test_incident(self):
3964         d = self.POST("/report_incident", details="eek")
3965         def _done(res):
3966             self.failIfIn("<html>", res)
3967             self.failUnlessIn("Thank you for your report!", res)
3968         d.addCallback(_done)
3969         return d
3970
3971     def test_static(self):
3972         webdir = os.path.join(self.staticdir, "subdir")
3973         fileutil.make_dirs(webdir)
3974         f = open(os.path.join(webdir, "hello.txt"), "wb")
3975         f.write("hello")
3976         f.close()
3977
3978         d = self.GET("/static/subdir/hello.txt")
3979         def _check(res):
3980             self.failUnlessReallyEqual(res, "hello")
3981         d.addCallback(_check)
3982         return d
3983
3984
3985 class IntroducerWeb(unittest.TestCase):
3986     def setUp(self):
3987         self.node = None
3988
3989     def tearDown(self):
3990         d = defer.succeed(None)
3991         if self.node:
3992             d.addCallback(lambda ign: self.node.stopService())
3993         d.addCallback(flushEventualQueue)
3994         return d
3995
3996     def test_welcome(self):
3997         basedir = "web.IntroducerWeb.test_welcome"
3998         os.mkdir(basedir)
3999         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4000         self.node = IntroducerNode(basedir)
4001         self.ws = self.node.getServiceNamed("webish")
4002
4003         d = fireEventually(None)
4004         d.addCallback(lambda ign: self.node.startService())
4005         d.addCallback(lambda ign: self.node.when_tub_ready())
4006
4007         d.addCallback(lambda ign: self.GET("/"))
4008         def _check(res):
4009             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4010             self.failUnlessIn(FAVICON_MARKUP, res)
4011         d.addCallback(_check)
4012         return d
4013
4014     def GET(self, urlpath, followRedirect=False, return_response=False,
4015             **kwargs):
4016         # if return_response=True, this fires with (data, statuscode,
4017         # respheaders) instead of just data.
4018         assert not isinstance(urlpath, unicode)
4019         url = self.ws.getURL().rstrip('/') + urlpath
4020         factory = HTTPClientGETFactory(url, method="GET",
4021                                        followRedirect=followRedirect, **kwargs)
4022         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4023         d = factory.deferred
4024         def _got_data(data):
4025             return (data, factory.status, factory.response_headers)
4026         if return_response:
4027             d.addCallback(_got_data)
4028         return factory.deferred
4029
4030
4031 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4032     def test_load_file(self):
4033         # This will raise an exception unless a well-formed XML file is found under that name.
4034         common.getxmlfile('directory.xhtml').load()
4035
4036     def test_parse_replace_arg(self):
4037         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4038         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4039         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4040                                    "only-files")
4041         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4042                         common.parse_replace_arg, "only_fles")
4043
4044     def test_abbreviate_time(self):
4045         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4046         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4047         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4048         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4049         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4050         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4051
4052     def test_compute_rate(self):
4053         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4054         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4055         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4056         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4057         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4058         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4059         self.shouldFail(AssertionError, "test_compute_rate", "",
4060                         common.compute_rate, -100, 10)
4061         self.shouldFail(AssertionError, "test_compute_rate", "",
4062                         common.compute_rate, 100, -10)
4063
4064         # Sanity check
4065         rate = common.compute_rate(10*1000*1000, 1)
4066         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4067
4068     def test_abbreviate_rate(self):
4069         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4070         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4071         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4072         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4073
4074     def test_abbreviate_size(self):
4075         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4076         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4077         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4078         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4079         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4080
4081     def test_plural(self):
4082         def convert(s):
4083             return "%d second%s" % (s, status.plural(s))
4084         self.failUnlessReallyEqual(convert(0), "0 seconds")
4085         self.failUnlessReallyEqual(convert(1), "1 second")
4086         self.failUnlessReallyEqual(convert(2), "2 seconds")
4087         def convert2(s):
4088             return "has share%s: %s" % (status.plural(s), ",".join(s))
4089         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4090         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4091         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4092
4093
4094 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4095
4096     def CHECK(self, ign, which, args, clientnum=0):
4097         fileurl = self.fileurls[which]
4098         url = fileurl + "?" + args
4099         return self.GET(url, method="POST", clientnum=clientnum)
4100
4101     def test_filecheck(self):
4102         self.basedir = "web/Grid/filecheck"
4103         self.set_up_grid()
4104         c0 = self.g.clients[0]
4105         self.uris = {}
4106         DATA = "data" * 100
4107         d = c0.upload(upload.Data(DATA, convergence=""))
4108         def _stash_uri(ur, which):
4109             self.uris[which] = ur.uri
4110         d.addCallback(_stash_uri, "good")
4111         d.addCallback(lambda ign:
4112                       c0.upload(upload.Data(DATA+"1", convergence="")))
4113         d.addCallback(_stash_uri, "sick")
4114         d.addCallback(lambda ign:
4115                       c0.upload(upload.Data(DATA+"2", convergence="")))
4116         d.addCallback(_stash_uri, "dead")
4117         def _stash_mutable_uri(n, which):
4118             self.uris[which] = n.get_uri()
4119             assert isinstance(self.uris[which], str)
4120         d.addCallback(lambda ign:
4121             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4122         d.addCallback(_stash_mutable_uri, "corrupt")
4123         d.addCallback(lambda ign:
4124                       c0.upload(upload.Data("literal", convergence="")))
4125         d.addCallback(_stash_uri, "small")
4126         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4127         d.addCallback(_stash_mutable_uri, "smalldir")
4128
4129         def _compute_fileurls(ignored):
4130             self.fileurls = {}
4131             for which in self.uris:
4132                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4133         d.addCallback(_compute_fileurls)
4134
4135         def _clobber_shares(ignored):
4136             good_shares = self.find_uri_shares(self.uris["good"])
4137             self.failUnlessReallyEqual(len(good_shares), 10)
4138             sick_shares = self.find_uri_shares(self.uris["sick"])
4139             os.unlink(sick_shares[0][2])
4140             dead_shares = self.find_uri_shares(self.uris["dead"])
4141             for i in range(1, 10):
4142                 os.unlink(dead_shares[i][2])
4143             c_shares = self.find_uri_shares(self.uris["corrupt"])
4144             cso = CorruptShareOptions()
4145             cso.stdout = StringIO()
4146             cso.parseOptions([c_shares[0][2]])
4147             corrupt_share(cso)
4148         d.addCallback(_clobber_shares)
4149
4150         d.addCallback(self.CHECK, "good", "t=check")
4151         def _got_html_good(res):
4152             self.failUnlessIn("Healthy", res)
4153             self.failIfIn("Not Healthy", res)
4154             self.failUnlessIn(FAVICON_MARKUP, res)
4155         d.addCallback(_got_html_good)
4156         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4157         def _got_html_good_return_to(res):
4158             self.failUnlessIn("Healthy", res)
4159             self.failIfIn("Not Healthy", res)
4160             self.failUnlessIn('<a href="somewhere">Return to file', res)
4161         d.addCallback(_got_html_good_return_to)
4162         d.addCallback(self.CHECK, "good", "t=check&output=json")
4163         def _got_json_good(res):
4164             r = simplejson.loads(res)
4165             self.failUnlessEqual(r["summary"], "Healthy")
4166             self.failUnless(r["results"]["healthy"])
4167             self.failIf(r["results"]["needs-rebalancing"])
4168             self.failUnless(r["results"]["recoverable"])
4169         d.addCallback(_got_json_good)
4170
4171         d.addCallback(self.CHECK, "small", "t=check")
4172         def _got_html_small(res):
4173             self.failUnlessIn("Literal files are always healthy", res)
4174             self.failIfIn("Not Healthy", res)
4175         d.addCallback(_got_html_small)
4176         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4177         def _got_html_small_return_to(res):
4178             self.failUnlessIn("Literal files are always healthy", res)
4179             self.failIfIn("Not Healthy", res)
4180             self.failUnlessIn('<a href="somewhere">Return to file', res)
4181         d.addCallback(_got_html_small_return_to)
4182         d.addCallback(self.CHECK, "small", "t=check&output=json")
4183         def _got_json_small(res):
4184             r = simplejson.loads(res)
4185             self.failUnlessEqual(r["storage-index"], "")
4186             self.failUnless(r["results"]["healthy"])
4187         d.addCallback(_got_json_small)
4188
4189         d.addCallback(self.CHECK, "smalldir", "t=check")
4190         def _got_html_smalldir(res):
4191             self.failUnlessIn("Literal files are always healthy", res)
4192             self.failIfIn("Not Healthy", res)
4193         d.addCallback(_got_html_smalldir)
4194         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4195         def _got_json_smalldir(res):
4196             r = simplejson.loads(res)
4197             self.failUnlessEqual(r["storage-index"], "")
4198             self.failUnless(r["results"]["healthy"])
4199         d.addCallback(_got_json_smalldir)
4200
4201         d.addCallback(self.CHECK, "sick", "t=check")
4202         def _got_html_sick(res):
4203             self.failUnlessIn("Not Healthy", res)
4204         d.addCallback(_got_html_sick)
4205         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4206         def _got_json_sick(res):
4207             r = simplejson.loads(res)
4208             self.failUnlessEqual(r["summary"],
4209                                  "Not Healthy: 9 shares (enc 3-of-10)")
4210             self.failIf(r["results"]["healthy"])
4211             self.failIf(r["results"]["needs-rebalancing"])
4212             self.failUnless(r["results"]["recoverable"])
4213         d.addCallback(_got_json_sick)
4214
4215         d.addCallback(self.CHECK, "dead", "t=check")
4216         def _got_html_dead(res):
4217             self.failUnlessIn("Not Healthy", res)
4218         d.addCallback(_got_html_dead)
4219         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4220         def _got_json_dead(res):
4221             r = simplejson.loads(res)
4222             self.failUnlessEqual(r["summary"],
4223                                  "Not Healthy: 1 shares (enc 3-of-10)")
4224             self.failIf(r["results"]["healthy"])
4225             self.failIf(r["results"]["needs-rebalancing"])
4226             self.failIf(r["results"]["recoverable"])
4227         d.addCallback(_got_json_dead)
4228
4229         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4230         def _got_html_corrupt(res):
4231             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4232         d.addCallback(_got_html_corrupt)
4233         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4234         def _got_json_corrupt(res):
4235             r = simplejson.loads(res)
4236             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4237             self.failIf(r["results"]["healthy"])
4238             self.failUnless(r["results"]["recoverable"])
4239             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4240             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4241         d.addCallback(_got_json_corrupt)
4242
4243         d.addErrback(self.explain_web_error)
4244         return d
4245
4246     def test_repair_html(self):
4247         self.basedir = "web/Grid/repair_html"
4248         self.set_up_grid()
4249         c0 = self.g.clients[0]
4250         self.uris = {}
4251         DATA = "data" * 100
4252         d = c0.upload(upload.Data(DATA, convergence=""))
4253         def _stash_uri(ur, which):
4254             self.uris[which] = ur.uri
4255         d.addCallback(_stash_uri, "good")
4256         d.addCallback(lambda ign:
4257                       c0.upload(upload.Data(DATA+"1", convergence="")))
4258         d.addCallback(_stash_uri, "sick")
4259         d.addCallback(lambda ign:
4260                       c0.upload(upload.Data(DATA+"2", convergence="")))
4261         d.addCallback(_stash_uri, "dead")
4262         def _stash_mutable_uri(n, which):
4263             self.uris[which] = n.get_uri()
4264             assert isinstance(self.uris[which], str)
4265         d.addCallback(lambda ign:
4266             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4267         d.addCallback(_stash_mutable_uri, "corrupt")
4268
4269         def _compute_fileurls(ignored):
4270             self.fileurls = {}
4271             for which in self.uris:
4272                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4273         d.addCallback(_compute_fileurls)
4274
4275         def _clobber_shares(ignored):
4276             good_shares = self.find_uri_shares(self.uris["good"])
4277             self.failUnlessReallyEqual(len(good_shares), 10)
4278             sick_shares = self.find_uri_shares(self.uris["sick"])
4279             os.unlink(sick_shares[0][2])
4280             dead_shares = self.find_uri_shares(self.uris["dead"])
4281             for i in range(1, 10):
4282                 os.unlink(dead_shares[i][2])
4283             c_shares = self.find_uri_shares(self.uris["corrupt"])
4284             cso = CorruptShareOptions()
4285             cso.stdout = StringIO()
4286             cso.parseOptions([c_shares[0][2]])
4287             corrupt_share(cso)
4288         d.addCallback(_clobber_shares)
4289
4290         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4291         def _got_html_good(res):
4292             self.failUnlessIn("Healthy", res)
4293             self.failIfIn("Not Healthy", res)
4294             self.failUnlessIn("No repair necessary", res)
4295             self.failUnlessIn(FAVICON_MARKUP, res)
4296         d.addCallback(_got_html_good)
4297
4298         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4299         def _got_html_sick(res):
4300             self.failUnlessIn("Healthy : healthy", res)
4301             self.failIfIn("Not Healthy", res)
4302             self.failUnlessIn("Repair successful", res)
4303         d.addCallback(_got_html_sick)
4304
4305         # repair of a dead file will fail, of course, but it isn't yet
4306         # clear how this should be reported. Right now it shows up as
4307         # a "410 Gone".
4308         #
4309         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4310         #def _got_html_dead(res):
4311         #    print res
4312         #    self.failUnlessIn("Healthy : healthy", res)
4313         #    self.failIfIn("Not Healthy", res)
4314         #    self.failUnlessIn("No repair necessary", res)
4315         #d.addCallback(_got_html_dead)
4316
4317         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4318         def _got_html_corrupt(res):
4319             self.failUnlessIn("Healthy : Healthy", res)
4320             self.failIfIn("Not Healthy", res)
4321             self.failUnlessIn("Repair successful", res)
4322         d.addCallback(_got_html_corrupt)
4323
4324         d.addErrback(self.explain_web_error)
4325         return d
4326
4327     def test_repair_json(self):
4328         self.basedir = "web/Grid/repair_json"
4329         self.set_up_grid()
4330         c0 = self.g.clients[0]
4331         self.uris = {}
4332         DATA = "data" * 100
4333         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4334         def _stash_uri(ur, which):
4335             self.uris[which] = ur.uri
4336         d.addCallback(_stash_uri, "sick")
4337
4338         def _compute_fileurls(ignored):
4339             self.fileurls = {}
4340             for which in self.uris:
4341                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4342         d.addCallback(_compute_fileurls)
4343
4344         def _clobber_shares(ignored):
4345             sick_shares = self.find_uri_shares(self.uris["sick"])
4346             os.unlink(sick_shares[0][2])
4347         d.addCallback(_clobber_shares)
4348
4349         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4350         def _got_json_sick(res):
4351             r = simplejson.loads(res)
4352             self.failUnlessReallyEqual(r["repair-attempted"], True)
4353             self.failUnlessReallyEqual(r["repair-successful"], True)
4354             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4355                                  "Not Healthy: 9 shares (enc 3-of-10)")
4356             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4357             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4358             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4359         d.addCallback(_got_json_sick)
4360
4361         d.addErrback(self.explain_web_error)
4362         return d
4363
4364     def test_unknown(self, immutable=False):
4365         self.basedir = "web/Grid/unknown"
4366         if immutable:
4367             self.basedir = "web/Grid/unknown-immutable"
4368
4369         self.set_up_grid()
4370         c0 = self.g.clients[0]
4371         self.uris = {}
4372         self.fileurls = {}
4373
4374         # the future cap format may contain slashes, which must be tolerated
4375         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4376                                                            safe="")
4377
4378         if immutable:
4379             name = u"future-imm"
4380             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4381             d = c0.create_immutable_dirnode({name: (future_node, {})})
4382         else:
4383             name = u"future"
4384             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4385             d = c0.create_dirnode()
4386
4387         def _stash_root_and_create_file(n):
4388             self.rootnode = n
4389             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4390             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4391             if not immutable:
4392                 return self.rootnode.set_node(name, future_node)
4393         d.addCallback(_stash_root_and_create_file)
4394
4395         # make sure directory listing tolerates unknown nodes
4396         d.addCallback(lambda ign: self.GET(self.rooturl))
4397         def _check_directory_html(res, expected_type_suffix):
4398             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4399                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4400                                  re.DOTALL)
4401             self.failUnless(re.search(pattern, res), res)
4402             # find the More Info link for name, should be relative
4403             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4404             info_url = mo.group(1)
4405             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4406         if immutable:
4407             d.addCallback(_check_directory_html, "-IMM")
4408         else:
4409             d.addCallback(_check_directory_html, "")
4410
4411         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4412         def _check_directory_json(res, expect_rw_uri):
4413             data = simplejson.loads(res)
4414             self.failUnlessEqual(data[0], "dirnode")
4415             f = data[1]["children"][name]
4416             self.failUnlessEqual(f[0], "unknown")
4417             if expect_rw_uri:
4418                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4419             else:
4420                 self.failIfIn("rw_uri", f[1])
4421             if immutable:
4422                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4423             else:
4424                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4425             self.failUnlessIn("metadata", f[1])
4426         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4427
4428         def _check_info(res, expect_rw_uri, expect_ro_uri):
4429             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4430             if expect_rw_uri:
4431                 self.failUnlessIn(unknown_rwcap, res)
4432             if expect_ro_uri:
4433                 if immutable:
4434                     self.failUnlessIn(unknown_immcap, res)
4435                 else:
4436                     self.failUnlessIn(unknown_rocap, res)
4437             else:
4438                 self.failIfIn(unknown_rocap, res)
4439             self.failIfIn("Raw data as", res)
4440             self.failIfIn("Directory writecap", res)
4441             self.failIfIn("Checker Operations", res)
4442             self.failIfIn("Mutable File Operations", res)
4443             self.failIfIn("Directory Operations", res)
4444
4445         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4446         # why they fail. Possibly related to ticket #922.
4447
4448         d.addCallback(lambda ign: self.GET(expected_info_url))
4449         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4450         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4451         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4452
4453         def _check_json(res, expect_rw_uri):
4454             data = simplejson.loads(res)
4455             self.failUnlessEqual(data[0], "unknown")
4456             if expect_rw_uri:
4457                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4458             else:
4459                 self.failIfIn("rw_uri", data[1])
4460
4461             if immutable:
4462                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4463                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4464             elif expect_rw_uri:
4465                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4466                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4467             else:
4468                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4469                 self.failIfIn("mutable", data[1])
4470
4471             # TODO: check metadata contents
4472             self.failUnlessIn("metadata", data[1])
4473
4474         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4475         d.addCallback(_check_json, expect_rw_uri=not immutable)
4476
4477         # and make sure that a read-only version of the directory can be
4478         # rendered too. This version will not have unknown_rwcap, whether
4479         # or not future_node was immutable.
4480         d.addCallback(lambda ign: self.GET(self.rourl))
4481         if immutable:
4482             d.addCallback(_check_directory_html, "-IMM")
4483         else:
4484             d.addCallback(_check_directory_html, "-RO")
4485
4486         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4487         d.addCallback(_check_directory_json, expect_rw_uri=False)
4488
4489         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4490         d.addCallback(_check_json, expect_rw_uri=False)
4491
4492         # TODO: check that getting t=info from the Info link in the ro directory
4493         # works, and does not include the writecap URI.
4494         return d
4495
4496     def test_immutable_unknown(self):
4497         return self.test_unknown(immutable=True)
4498
4499     def test_mutant_dirnodes_are_omitted(self):
4500         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4501
4502         self.set_up_grid()
4503         c = self.g.clients[0]
4504         nm = c.nodemaker
4505         self.uris = {}
4506         self.fileurls = {}
4507
4508         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4509         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4510         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4511
4512         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4513         # test the dirnode and web layers separately.
4514
4515         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4516         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4517         # When the directory is read, the mutants should be silently disposed of, leaving
4518         # their lonely sibling.
4519         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4520         # because immutable directories don't have a writecap and therefore that field
4521         # isn't (and can't be) decrypted.
4522         # TODO: The field still exists in the netstring. Technically we should check what
4523         # happens if something is put there (_unpack_contents should raise ValueError),
4524         # but that can wait.
4525
4526         lonely_child = nm.create_from_cap(lonely_uri)
4527         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4528         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4529
4530         def _by_hook_or_by_crook():
4531             return True
4532         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4533             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4534
4535         mutant_write_in_ro_child.get_write_uri    = lambda: None
4536         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4537
4538         kids = {u"lonely":      (lonely_child, {}),
4539                 u"ro":          (mutant_ro_child, {}),
4540                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4541                 }
4542         d = c.create_immutable_dirnode(kids)
4543
4544         def _created(dn):
4545             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4546             self.failIf(dn.is_mutable())
4547             self.failUnless(dn.is_readonly())
4548             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4549             self.failIf(hasattr(dn._node, 'get_writekey'))
4550             rep = str(dn)
4551             self.failUnlessIn("RO-IMM", rep)
4552             cap = dn.get_cap()
4553             self.failUnlessIn("CHK", cap.to_string())
4554             self.cap = cap
4555             self.rootnode = dn
4556             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4557             return download_to_data(dn._node)
4558         d.addCallback(_created)
4559
4560         def _check_data(data):
4561             # Decode the netstring representation of the directory to check that all children
4562             # are present. This is a bit of an abstraction violation, but there's not really
4563             # any other way to do it given that the real DirectoryNode._unpack_contents would
4564             # strip the mutant children out (which is what we're trying to test, later).
4565             position = 0
4566             numkids = 0
4567             while position < len(data):
4568                 entries, position = split_netstring(data, 1, position)
4569                 entry = entries[0]
4570                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4571                 name = name_utf8.decode("utf-8")
4572                 self.failUnlessEqual(rwcapdata, "")
4573                 self.failUnlessIn(name, kids)
4574                 (expected_child, ign) = kids[name]
4575                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4576                 numkids += 1
4577
4578             self.failUnlessReallyEqual(numkids, 3)
4579             return self.rootnode.list()
4580         d.addCallback(_check_data)
4581
4582         # Now when we use the real directory listing code, the mutants should be absent.
4583         def _check_kids(children):
4584             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4585             lonely_node, lonely_metadata = children[u"lonely"]
4586
4587             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4588             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4589         d.addCallback(_check_kids)
4590
4591         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4592         d.addCallback(lambda n: n.list())
4593         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4594
4595         # Make sure the lonely child can be listed in HTML...
4596         d.addCallback(lambda ign: self.GET(self.rooturl))
4597         def _check_html(res):
4598             self.failIfIn("URI:SSK", res)
4599             get_lonely = "".join([r'<td>FILE</td>',
4600                                   r'\s+<td>',
4601                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4602                                   r'</td>',
4603                                   r'\s+<td align="right">%d</td>' % len("one"),
4604                                   ])
4605             self.failUnless(re.search(get_lonely, res), res)
4606
4607             # find the More Info link for name, should be relative
4608             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4609             info_url = mo.group(1)
4610             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4611         d.addCallback(_check_html)
4612
4613         # ... and in JSON.
4614         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4615         def _check_json(res):
4616             data = simplejson.loads(res)
4617             self.failUnlessEqual(data[0], "dirnode")
4618             listed_children = data[1]["children"]
4619             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4620             ll_type, ll_data = listed_children[u"lonely"]
4621             self.failUnlessEqual(ll_type, "filenode")
4622             self.failIfIn("rw_uri", ll_data)
4623             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4624         d.addCallback(_check_json)
4625         return d
4626
4627     def test_deep_check(self):
4628         self.basedir = "web/Grid/deep_check"
4629         self.set_up_grid()
4630         c0 = self.g.clients[0]
4631         self.uris = {}
4632         self.fileurls = {}
4633         DATA = "data" * 100
4634         d = c0.create_dirnode()
4635         def _stash_root_and_create_file(n):
4636             self.rootnode = n
4637             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4638             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4639         d.addCallback(_stash_root_and_create_file)
4640         def _stash_uri(fn, which):
4641             self.uris[which] = fn.get_uri()
4642             return fn
4643         d.addCallback(_stash_uri, "good")
4644         d.addCallback(lambda ign:
4645                       self.rootnode.add_file(u"small",
4646                                              upload.Data("literal",
4647                                                         convergence="")))
4648         d.addCallback(_stash_uri, "small")
4649         d.addCallback(lambda ign:
4650                       self.rootnode.add_file(u"sick",
4651                                              upload.Data(DATA+"1",
4652                                                         convergence="")))
4653         d.addCallback(_stash_uri, "sick")
4654
4655         # this tests that deep-check and stream-manifest will ignore
4656         # UnknownNode instances. Hopefully this will also cover deep-stats.
4657         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4658         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4659
4660         def _clobber_shares(ignored):
4661             self.delete_shares_numbered(self.uris["sick"], [0,1])
4662         d.addCallback(_clobber_shares)
4663
4664         # root
4665         # root/good
4666         # root/small
4667         # root/sick
4668         # root/future
4669
4670         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4671         def _done(res):
4672             try:
4673                 units = [simplejson.loads(line)
4674                          for line in res.splitlines()
4675                          if line]
4676             except ValueError:
4677                 print "response is:", res
4678                 print "undecodeable line was '%s'" % line
4679                 raise
4680             self.failUnlessReallyEqual(len(units), 5+1)
4681             # should be parent-first
4682             u0 = units[0]
4683             self.failUnlessEqual(u0["path"], [])
4684             self.failUnlessEqual(u0["type"], "directory")
4685             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4686             u0cr = u0["check-results"]
4687             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4688
4689             ugood = [u for u in units
4690                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4691             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4692             ugoodcr = ugood["check-results"]
4693             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4694
4695             stats = units[-1]
4696             self.failUnlessEqual(stats["type"], "stats")
4697             s = stats["stats"]
4698             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4699             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4700             self.failUnlessReallyEqual(s["count-directories"], 1)
4701             self.failUnlessReallyEqual(s["count-unknown"], 1)
4702         d.addCallback(_done)
4703
4704         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4705         def _check_manifest(res):
4706             self.failUnless(res.endswith("\n"))
4707             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4708             self.failUnlessReallyEqual(len(units), 5+1)
4709             self.failUnlessEqual(units[-1]["type"], "stats")
4710             first = units[0]
4711             self.failUnlessEqual(first["path"], [])
4712             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4713             self.failUnlessEqual(first["type"], "directory")
4714             stats = units[-1]["stats"]
4715             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4716             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4717             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4718             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4719             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4720         d.addCallback(_check_manifest)
4721
4722         # now add root/subdir and root/subdir/grandchild, then make subdir
4723         # unrecoverable, then see what happens
4724
4725         d.addCallback(lambda ign:
4726                       self.rootnode.create_subdirectory(u"subdir"))
4727         d.addCallback(_stash_uri, "subdir")
4728         d.addCallback(lambda subdir_node:
4729                       subdir_node.add_file(u"grandchild",
4730                                            upload.Data(DATA+"2",
4731                                                        convergence="")))
4732         d.addCallback(_stash_uri, "grandchild")
4733
4734         d.addCallback(lambda ign:
4735                       self.delete_shares_numbered(self.uris["subdir"],
4736                                                   range(1, 10)))
4737
4738         # root
4739         # root/good
4740         # root/small
4741         # root/sick
4742         # root/future
4743         # root/subdir [unrecoverable]
4744         # root/subdir/grandchild
4745
4746         # how should a streaming-JSON API indicate fatal error?
4747         # answer: emit ERROR: instead of a JSON string
4748
4749         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4750         def _check_broken_manifest(res):
4751             lines = res.splitlines()
4752             error_lines = [i
4753                            for (i,line) in enumerate(lines)
4754                            if line.startswith("ERROR:")]
4755             if not error_lines:
4756                 self.fail("no ERROR: in output: %s" % (res,))
4757             first_error = error_lines[0]
4758             error_line = lines[first_error]
4759             error_msg = lines[first_error+1:]
4760             error_msg_s = "\n".join(error_msg) + "\n"
4761             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4762                               error_line)
4763             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4764             units = [simplejson.loads(line) for line in lines[:first_error]]
4765             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4766             last_unit = units[-1]
4767             self.failUnlessEqual(last_unit["path"], ["subdir"])
4768         d.addCallback(_check_broken_manifest)
4769
4770         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4771         def _check_broken_deepcheck(res):
4772             lines = res.splitlines()
4773             error_lines = [i
4774                            for (i,line) in enumerate(lines)
4775                            if line.startswith("ERROR:")]
4776             if not error_lines:
4777                 self.fail("no ERROR: in output: %s" % (res,))
4778             first_error = error_lines[0]
4779             error_line = lines[first_error]
4780             error_msg = lines[first_error+1:]
4781             error_msg_s = "\n".join(error_msg) + "\n"
4782             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4783                               error_line)
4784             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4785             units = [simplejson.loads(line) for line in lines[:first_error]]
4786             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4787             last_unit = units[-1]
4788             self.failUnlessEqual(last_unit["path"], ["subdir"])
4789             r = last_unit["check-results"]["results"]
4790             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4791             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4792             self.failUnlessReallyEqual(r["recoverable"], False)
4793         d.addCallback(_check_broken_deepcheck)
4794
4795         d.addErrback(self.explain_web_error)
4796         return d
4797
4798     def test_deep_check_and_repair(self):
4799         self.basedir = "web/Grid/deep_check_and_repair"
4800         self.set_up_grid()
4801         c0 = self.g.clients[0]
4802         self.uris = {}
4803         self.fileurls = {}
4804         DATA = "data" * 100
4805         d = c0.create_dirnode()
4806         def _stash_root_and_create_file(n):
4807             self.rootnode = n
4808             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4809             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4810         d.addCallback(_stash_root_and_create_file)
4811         def _stash_uri(fn, which):
4812             self.uris[which] = fn.get_uri()
4813         d.addCallback(_stash_uri, "good")
4814         d.addCallback(lambda ign:
4815                       self.rootnode.add_file(u"small",
4816                                              upload.Data("literal",
4817                                                         convergence="")))
4818         d.addCallback(_stash_uri, "small")
4819         d.addCallback(lambda ign:
4820                       self.rootnode.add_file(u"sick",
4821                                              upload.Data(DATA+"1",
4822                                                         convergence="")))
4823         d.addCallback(_stash_uri, "sick")
4824         #d.addCallback(lambda ign:
4825         #              self.rootnode.add_file(u"dead",
4826         #                                     upload.Data(DATA+"2",
4827         #                                                convergence="")))
4828         #d.addCallback(_stash_uri, "dead")
4829
4830         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4831         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4832         #d.addCallback(_stash_uri, "corrupt")
4833
4834         def _clobber_shares(ignored):
4835             good_shares = self.find_uri_shares(self.uris["good"])
4836             self.failUnlessReallyEqual(len(good_shares), 10)
4837             sick_shares = self.find_uri_shares(self.uris["sick"])
4838             os.unlink(sick_shares[0][2])
4839             #dead_shares = self.find_uri_shares(self.uris["dead"])
4840             #for i in range(1, 10):
4841             #    os.unlink(dead_shares[i][2])
4842
4843             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4844             #cso = CorruptShareOptions()
4845             #cso.stdout = StringIO()
4846             #cso.parseOptions([c_shares[0][2]])
4847             #corrupt_share(cso)
4848         d.addCallback(_clobber_shares)
4849
4850         # root
4851         # root/good   CHK, 10 shares
4852         # root/small  LIT
4853         # root/sick   CHK, 9 shares
4854
4855         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4856         def _done(res):
4857             units = [simplejson.loads(line)
4858                      for line in res.splitlines()
4859                      if line]
4860             self.failUnlessReallyEqual(len(units), 4+1)
4861             # should be parent-first
4862             u0 = units[0]
4863             self.failUnlessEqual(u0["path"], [])
4864             self.failUnlessEqual(u0["type"], "directory")
4865             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4866             u0crr = u0["check-and-repair-results"]
4867             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4868             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4869
4870             ugood = [u for u in units
4871                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4872             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4873             ugoodcrr = ugood["check-and-repair-results"]
4874             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4875             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4876
4877             usick = [u for u in units
4878                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4879             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4880             usickcrr = usick["check-and-repair-results"]
4881             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4882             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4883             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4884             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4885
4886             stats = units[-1]
4887             self.failUnlessEqual(stats["type"], "stats")
4888             s = stats["stats"]
4889             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4890             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4891             self.failUnlessReallyEqual(s["count-directories"], 1)
4892         d.addCallback(_done)
4893
4894         d.addErrback(self.explain_web_error)
4895         return d
4896
4897     def _count_leases(self, ignored, which):
4898         u = self.uris[which]
4899         shares = self.find_uri_shares(u)
4900         lease_counts = []
4901         for shnum, serverid, fn in shares:
4902             sf = get_share_file(fn)
4903             num_leases = len(list(sf.get_leases()))
4904             lease_counts.append( (fn, num_leases) )
4905         return lease_counts
4906
4907     def _assert_leasecount(self, lease_counts, expected):
4908         for (fn, num_leases) in lease_counts:
4909             if num_leases != expected:
4910                 self.fail("expected %d leases, have %d, on %s" %
4911                           (expected, num_leases, fn))
4912
4913     def test_add_lease(self):
4914         self.basedir = "web/Grid/add_lease"
4915         self.set_up_grid(num_clients=2)
4916         c0 = self.g.clients[0]
4917         self.uris = {}
4918         DATA = "data" * 100
4919         d = c0.upload(upload.Data(DATA, convergence=""))
4920         def _stash_uri(ur, which):
4921             self.uris[which] = ur.uri
4922         d.addCallback(_stash_uri, "one")
4923         d.addCallback(lambda ign:
4924                       c0.upload(upload.Data(DATA+"1", convergence="")))
4925         d.addCallback(_stash_uri, "two")
4926         def _stash_mutable_uri(n, which):
4927             self.uris[which] = n.get_uri()
4928             assert isinstance(self.uris[which], str)
4929         d.addCallback(lambda ign:
4930             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4931         d.addCallback(_stash_mutable_uri, "mutable")
4932
4933         def _compute_fileurls(ignored):
4934             self.fileurls = {}
4935             for which in self.uris:
4936                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4937         d.addCallback(_compute_fileurls)
4938
4939         d.addCallback(self._count_leases, "one")
4940         d.addCallback(self._assert_leasecount, 1)
4941         d.addCallback(self._count_leases, "two")
4942         d.addCallback(self._assert_leasecount, 1)
4943         d.addCallback(self._count_leases, "mutable")
4944         d.addCallback(self._assert_leasecount, 1)
4945
4946         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4947         def _got_html_good(res):
4948             self.failUnlessIn("Healthy", res)
4949             self.failIfIn("Not Healthy", res)
4950         d.addCallback(_got_html_good)
4951
4952         d.addCallback(self._count_leases, "one")
4953         d.addCallback(self._assert_leasecount, 1)
4954         d.addCallback(self._count_leases, "two")
4955         d.addCallback(self._assert_leasecount, 1)
4956         d.addCallback(self._count_leases, "mutable")
4957         d.addCallback(self._assert_leasecount, 1)
4958
4959         # this CHECK uses the original client, which uses the same
4960         # lease-secrets, so it will just renew the original lease
4961         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4962         d.addCallback(_got_html_good)
4963
4964         d.addCallback(self._count_leases, "one")
4965         d.addCallback(self._assert_leasecount, 1)
4966         d.addCallback(self._count_leases, "two")
4967         d.addCallback(self._assert_leasecount, 1)
4968         d.addCallback(self._count_leases, "mutable")
4969         d.addCallback(self._assert_leasecount, 1)
4970
4971         # this CHECK uses an alternate client, which adds a second lease
4972         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4973         d.addCallback(_got_html_good)
4974
4975         d.addCallback(self._count_leases, "one")
4976         d.addCallback(self._assert_leasecount, 2)
4977         d.addCallback(self._count_leases, "two")
4978         d.addCallback(self._assert_leasecount, 1)
4979         d.addCallback(self._count_leases, "mutable")
4980         d.addCallback(self._assert_leasecount, 1)
4981
4982         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4983         d.addCallback(_got_html_good)
4984
4985         d.addCallback(self._count_leases, "one")
4986         d.addCallback(self._assert_leasecount, 2)
4987         d.addCallback(self._count_leases, "two")
4988         d.addCallback(self._assert_leasecount, 1)
4989         d.addCallback(self._count_leases, "mutable")
4990         d.addCallback(self._assert_leasecount, 1)
4991
4992         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4993                       clientnum=1)
4994         d.addCallback(_got_html_good)
4995
4996         d.addCallback(self._count_leases, "one")
4997         d.addCallback(self._assert_leasecount, 2)
4998         d.addCallback(self._count_leases, "two")
4999         d.addCallback(self._assert_leasecount, 1)
5000         d.addCallback(self._count_leases, "mutable")
5001         d.addCallback(self._assert_leasecount, 2)
5002
5003         d.addErrback(self.explain_web_error)
5004         return d
5005
5006     def test_deep_add_lease(self):
5007         self.basedir = "web/Grid/deep_add_lease"
5008         self.set_up_grid(num_clients=2)
5009         c0 = self.g.clients[0]
5010         self.uris = {}
5011         self.fileurls = {}
5012         DATA = "data" * 100
5013         d = c0.create_dirnode()
5014         def _stash_root_and_create_file(n):
5015             self.rootnode = n
5016             self.uris["root"] = n.get_uri()
5017             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5018             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5019         d.addCallback(_stash_root_and_create_file)
5020         def _stash_uri(fn, which):
5021             self.uris[which] = fn.get_uri()
5022         d.addCallback(_stash_uri, "one")
5023         d.addCallback(lambda ign:
5024                       self.rootnode.add_file(u"small",
5025                                              upload.Data("literal",
5026                                                         convergence="")))
5027         d.addCallback(_stash_uri, "small")
5028
5029         d.addCallback(lambda ign:
5030             c0.create_mutable_file(publish.MutableData("mutable")))
5031         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5032         d.addCallback(_stash_uri, "mutable")
5033
5034         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5035         def _done(res):
5036             units = [simplejson.loads(line)
5037                      for line in res.splitlines()
5038                      if line]
5039             # root, one, small, mutable,   stats
5040             self.failUnlessReallyEqual(len(units), 4+1)
5041         d.addCallback(_done)
5042
5043         d.addCallback(self._count_leases, "root")
5044         d.addCallback(self._assert_leasecount, 1)
5045         d.addCallback(self._count_leases, "one")
5046         d.addCallback(self._assert_leasecount, 1)
5047         d.addCallback(self._count_leases, "mutable")
5048         d.addCallback(self._assert_leasecount, 1)
5049
5050         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5051         d.addCallback(_done)
5052
5053         d.addCallback(self._count_leases, "root")
5054         d.addCallback(self._assert_leasecount, 1)
5055         d.addCallback(self._count_leases, "one")
5056         d.addCallback(self._assert_leasecount, 1)
5057         d.addCallback(self._count_leases, "mutable")
5058         d.addCallback(self._assert_leasecount, 1)
5059
5060         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5061                       clientnum=1)
5062         d.addCallback(_done)
5063
5064         d.addCallback(self._count_leases, "root")
5065         d.addCallback(self._assert_leasecount, 2)
5066         d.addCallback(self._count_leases, "one")
5067         d.addCallback(self._assert_leasecount, 2)
5068         d.addCallback(self._count_leases, "mutable")
5069         d.addCallback(self._assert_leasecount, 2)
5070
5071         d.addErrback(self.explain_web_error)
5072         return d
5073
5074
5075     def test_exceptions(self):
5076         self.basedir = "web/Grid/exceptions"
5077         self.set_up_grid(num_clients=1, num_servers=2)
5078         c0 = self.g.clients[0]
5079         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5080         self.fileurls = {}
5081         DATA = "data" * 100
5082         d = c0.create_dirnode()
5083         def _stash_root(n):
5084             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5085             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5086             return n
5087         d.addCallback(_stash_root)
5088         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5089         def _stash_bad(ur):
5090             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5091             self.delete_shares_numbered(ur.uri, range(1,10))
5092
5093             u = uri.from_string(ur.uri)
5094             u.key = testutil.flip_bit(u.key, 0)
5095             baduri = u.to_string()
5096             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5097         d.addCallback(_stash_bad)
5098         d.addCallback(lambda ign: c0.create_dirnode())
5099         def _mangle_dirnode_1share(n):
5100             u = n.get_uri()
5101             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5102             self.fileurls["dir-1share-json"] = url + "?t=json"
5103             self.delete_shares_numbered(u, range(1,10))
5104         d.addCallback(_mangle_dirnode_1share)
5105         d.addCallback(lambda ign: c0.create_dirnode())
5106         def _mangle_dirnode_0share(n):
5107             u = n.get_uri()
5108             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5109             self.fileurls["dir-0share-json"] = url + "?t=json"
5110             self.delete_shares_numbered(u, range(0,10))
5111         d.addCallback(_mangle_dirnode_0share)
5112
5113         # NotEnoughSharesError should be reported sensibly, with a
5114         # text/plain explanation of the problem, and perhaps some
5115         # information on which shares *could* be found.
5116
5117         d.addCallback(lambda ignored:
5118                       self.shouldHTTPError("GET unrecoverable",
5119                                            410, "Gone", "NoSharesError",
5120                                            self.GET, self.fileurls["0shares"]))
5121         def _check_zero_shares(body):
5122             self.failIfIn("<html>", body)
5123             body = " ".join(body.strip().split())
5124             exp = ("NoSharesError: no shares could be found. "
5125                    "Zero shares usually indicates a corrupt URI, or that "
5126                    "no servers were connected, but it might also indicate "
5127                    "severe corruption. You should perform a filecheck on "
5128                    "this object to learn more. The full error message is: "
5129                    "no shares (need 3). Last failure: None")
5130             self.failUnlessReallyEqual(exp, body)
5131         d.addCallback(_check_zero_shares)
5132
5133
5134         d.addCallback(lambda ignored:
5135                       self.shouldHTTPError("GET 1share",
5136                                            410, "Gone", "NotEnoughSharesError",
5137                                            self.GET, self.fileurls["1share"]))
5138         def _check_one_share(body):
5139             self.failIfIn("<html>", body)
5140             body = " ".join(body.strip().split())
5141             msgbase = ("NotEnoughSharesError: This indicates that some "
5142                        "servers were unavailable, or that shares have been "
5143                        "lost to server departure, hard drive failure, or disk "
5144                        "corruption. You should perform a filecheck on "
5145                        "this object to learn more. The full error message is:"
5146                        )
5147             msg1 = msgbase + (" ran out of shares:"
5148                               " complete=sh0"
5149                               " pending="
5150                               " overdue= unused= need 3. Last failure: None")
5151             msg2 = msgbase + (" ran out of shares:"
5152                               " complete="
5153                               " pending=Share(sh0-on-xgru5)"
5154                               " overdue= unused= need 3. Last failure: None")
5155             self.failUnless(body == msg1 or body == msg2, body)
5156         d.addCallback(_check_one_share)
5157
5158         d.addCallback(lambda ignored:
5159                       self.shouldHTTPError("GET imaginary",
5160                                            404, "Not Found", None,
5161                                            self.GET, self.fileurls["imaginary"]))
5162         def _missing_child(body):
5163             self.failUnlessIn("No such child: imaginary", body)
5164         d.addCallback(_missing_child)
5165
5166         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5167         def _check_0shares_dir_html(body):
5168             self.failUnlessIn("<html>", body)
5169             # we should see the regular page, but without the child table or
5170             # the dirops forms
5171             body = " ".join(body.strip().split())
5172             self.failUnlessIn('href="?t=info">More info on this directory',
5173                               body)
5174             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5175                    "could not be retrieved, because there were insufficient "
5176                    "good shares. This might indicate that no servers were "
5177                    "connected, insufficient servers were connected, the URI "
5178                    "was corrupt, or that shares have been lost due to server "
5179                    "departure, hard drive failure, or disk corruption. You "
5180                    "should perform a filecheck on this object to learn more.")
5181             self.failUnlessIn(exp, body)
5182             self.failUnlessIn("No upload forms: directory is unreadable", body)
5183         d.addCallback(_check_0shares_dir_html)
5184
5185         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5186         def _check_1shares_dir_html(body):
5187             # at some point, we'll split UnrecoverableFileError into 0-shares
5188             # and some-shares like we did for immutable files (since there
5189             # are different sorts of advice to offer in each case). For now,
5190             # they present the same way.
5191             self.failUnlessIn("<html>", body)
5192             body = " ".join(body.strip().split())
5193             self.failUnlessIn('href="?t=info">More info on this directory',
5194                               body)
5195             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5196                    "could not be retrieved, because there were insufficient "
5197                    "good shares. This might indicate that no servers were "
5198                    "connected, insufficient servers were connected, the URI "
5199                    "was corrupt, or that shares have been lost due to server "
5200                    "departure, hard drive failure, or disk corruption. You "
5201                    "should perform a filecheck on this object to learn more.")
5202             self.failUnlessIn(exp, body)
5203             self.failUnlessIn("No upload forms: directory is unreadable", body)
5204         d.addCallback(_check_1shares_dir_html)
5205
5206         d.addCallback(lambda ignored:
5207                       self.shouldHTTPError("GET dir-0share-json",
5208                                            410, "Gone", "UnrecoverableFileError",
5209                                            self.GET,
5210                                            self.fileurls["dir-0share-json"]))
5211         def _check_unrecoverable_file(body):
5212             self.failIfIn("<html>", body)
5213             body = " ".join(body.strip().split())
5214             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5215                    "could not be retrieved, because there were insufficient "
5216                    "good shares. This might indicate that no servers were "
5217                    "connected, insufficient servers were connected, the URI "
5218                    "was corrupt, or that shares have been lost due to server "
5219                    "departure, hard drive failure, or disk corruption. You "
5220                    "should perform a filecheck on this object to learn more.")
5221             self.failUnlessReallyEqual(exp, body)
5222         d.addCallback(_check_unrecoverable_file)
5223
5224         d.addCallback(lambda ignored:
5225                       self.shouldHTTPError("GET dir-1share-json",
5226                                            410, "Gone", "UnrecoverableFileError",
5227                                            self.GET,
5228                                            self.fileurls["dir-1share-json"]))
5229         d.addCallback(_check_unrecoverable_file)
5230
5231         d.addCallback(lambda ignored:
5232                       self.shouldHTTPError("GET imaginary",
5233                                            404, "Not Found", None,
5234                                            self.GET, self.fileurls["imaginary"]))
5235
5236         # attach a webapi child that throws a random error, to test how it
5237         # gets rendered.
5238         w = c0.getServiceNamed("webish")
5239         w.root.putChild("ERRORBOOM", ErrorBoom())
5240
5241         # "Accept: */*" :        should get a text/html stack trace
5242         # "Accept: text/plain" : should get a text/plain stack trace
5243         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5244         # no Accept header:      should get a text/html stack trace
5245
5246         d.addCallback(lambda ignored:
5247                       self.shouldHTTPError("GET errorboom_html",
5248                                            500, "Internal Server Error", None,
5249                                            self.GET, "ERRORBOOM",
5250                                            headers={"accept": ["*/*"]}))
5251         def _internal_error_html1(body):
5252             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5253         d.addCallback(_internal_error_html1)
5254
5255         d.addCallback(lambda ignored:
5256                       self.shouldHTTPError("GET errorboom_text",
5257                                            500, "Internal Server Error", None,
5258                                            self.GET, "ERRORBOOM",
5259                                            headers={"accept": ["text/plain"]}))
5260         def _internal_error_text2(body):
5261             self.failIfIn("<html>", body)
5262             self.failUnless(body.startswith("Traceback "), body)
5263         d.addCallback(_internal_error_text2)
5264
5265         CLI_accepts = "text/plain, application/octet-stream"
5266         d.addCallback(lambda ignored:
5267                       self.shouldHTTPError("GET errorboom_text",
5268                                            500, "Internal Server Error", None,
5269                                            self.GET, "ERRORBOOM",
5270                                            headers={"accept": [CLI_accepts]}))
5271         def _internal_error_text3(body):
5272             self.failIfIn("<html>", body)
5273             self.failUnless(body.startswith("Traceback "), body)
5274         d.addCallback(_internal_error_text3)
5275
5276         d.addCallback(lambda ignored:
5277                       self.shouldHTTPError("GET errorboom_text",
5278                                            500, "Internal Server Error", None,
5279                                            self.GET, "ERRORBOOM"))
5280         def _internal_error_html4(body):
5281             self.failUnlessIn("<html>", body)
5282         d.addCallback(_internal_error_html4)
5283
5284         def _flush_errors(res):
5285             # Trial: please ignore the CompletelyUnhandledError in the logs
5286             self.flushLoggedErrors(CompletelyUnhandledError)
5287             return res
5288         d.addBoth(_flush_errors)
5289
5290         return d
5291
5292     def test_blacklist(self):
5293         # download from a blacklisted URI, get an error
5294         self.basedir = "web/Grid/blacklist"
5295         self.set_up_grid()
5296         c0 = self.g.clients[0]
5297         c0_basedir = c0.basedir
5298         fn = os.path.join(c0_basedir, "access.blacklist")
5299         self.uris = {}
5300         DATA = "off-limits " * 50
5301
5302         d = c0.upload(upload.Data(DATA, convergence=""))
5303         def _stash_uri_and_create_dir(ur):
5304             self.uri = ur.uri
5305             self.url = "uri/"+self.uri
5306             u = uri.from_string_filenode(self.uri)
5307             self.si = u.get_storage_index()
5308             childnode = c0.create_node_from_uri(self.uri, None)
5309             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5310         d.addCallback(_stash_uri_and_create_dir)
5311         def _stash_dir(node):
5312             self.dir_node = node
5313             self.dir_uri = node.get_uri()
5314             self.dir_url = "uri/"+self.dir_uri
5315         d.addCallback(_stash_dir)
5316         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5317         def _check_dir_html(body):
5318             self.failUnlessIn("<html>", body)
5319             self.failUnlessIn("blacklisted.txt</a>", body)
5320         d.addCallback(_check_dir_html)
5321         d.addCallback(lambda ign: self.GET(self.url))
5322         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5323
5324         def _blacklist(ign):
5325             f = open(fn, "w")
5326             f.write(" # this is a comment\n")
5327             f.write(" \n")
5328             f.write("\n") # also exercise blank lines
5329             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5330             f.close()
5331             # clients should be checking the blacklist each time, so we don't
5332             # need to restart the client
5333         d.addCallback(_blacklist)
5334         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5335                                                        403, "Forbidden",
5336                                                        "Access Prohibited: off-limits",
5337                                                        self.GET, self.url))
5338
5339         # We should still be able to list the parent directory, in HTML...
5340         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5341         def _check_dir_html2(body):
5342             self.failUnlessIn("<html>", body)
5343             self.failUnlessIn("blacklisted.txt</strike>", body)
5344         d.addCallback(_check_dir_html2)
5345
5346         # ... and in JSON (used by CLI).
5347         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5348         def _check_dir_json(res):
5349             data = simplejson.loads(res)
5350             self.failUnless(isinstance(data, list), data)
5351             self.failUnlessEqual(data[0], "dirnode")
5352             self.failUnless(isinstance(data[1], dict), data)
5353             self.failUnlessIn("children", data[1])
5354             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5355             childdata = data[1]["children"]["blacklisted.txt"]
5356             self.failUnless(isinstance(childdata, list), data)
5357             self.failUnlessEqual(childdata[0], "filenode")
5358             self.failUnless(isinstance(childdata[1], dict), data)
5359         d.addCallback(_check_dir_json)
5360
5361         def _unblacklist(ign):
5362             open(fn, "w").close()
5363             # the Blacklist object watches mtime to tell when the file has
5364             # changed, but on windows this test will run faster than the
5365             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5366             # to force a reload.
5367             self.g.clients[0].blacklist.last_mtime -= 2.0
5368         d.addCallback(_unblacklist)
5369
5370         # now a read should work
5371         d.addCallback(lambda ign: self.GET(self.url))
5372         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5373
5374         # read again to exercise the blacklist-is-unchanged logic
5375         d.addCallback(lambda ign: self.GET(self.url))
5376         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5377
5378         # now add a blacklisted directory, and make sure files under it are
5379         # refused too
5380         def _add_dir(ign):
5381             childnode = c0.create_node_from_uri(self.uri, None)
5382             return c0.create_dirnode({u"child": (childnode,{}) })
5383         d.addCallback(_add_dir)
5384         def _get_dircap(dn):
5385             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5386             self.dir_url_base = "uri/"+dn.get_write_uri()
5387             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5388             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5389             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5390             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5391         d.addCallback(_get_dircap)
5392         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5393         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5394         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5395         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5396         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5397         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5398         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5399         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5400         d.addCallback(lambda ign: self.GET(self.child_url))
5401         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5402
5403         def _block_dir(ign):
5404             f = open(fn, "w")
5405             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5406             f.close()
5407             self.g.clients[0].blacklist.last_mtime -= 2.0
5408         d.addCallback(_block_dir)
5409         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5410                                                        403, "Forbidden",
5411                                                        "Access Prohibited: dir-off-limits",
5412                                                        self.GET, self.dir_url_base))
5413         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5414                                                        403, "Forbidden",
5415                                                        "Access Prohibited: dir-off-limits",
5416                                                        self.GET, self.dir_url_json1))
5417         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5418                                                        403, "Forbidden",
5419                                                        "Access Prohibited: dir-off-limits",
5420                                                        self.GET, self.dir_url_json2))
5421         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5422                                                        403, "Forbidden",
5423                                                        "Access Prohibited: dir-off-limits",
5424                                                        self.GET, self.dir_url_json_ro))
5425         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5426                                                        403, "Forbidden",
5427                                                        "Access Prohibited: dir-off-limits",
5428                                                        self.GET, self.child_url))
5429         return d
5430
5431
5432 class CompletelyUnhandledError(Exception):
5433     pass
5434 class ErrorBoom(rend.Page):
5435     def beforeRender(self, ctx):
5436         raise CompletelyUnhandledError("whoops")