src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  46
  47
  48 class FakeStatsProvider:
  49     def get_stats(self):
  50         stats = {'stats': {}, 'counters': {}}
  51         return stats
  52
  53 class FakeNodeMaker(NodeMaker):
  54     encoding_params = {
  55         'k': 3,
  56         'n': 10,
  57         'happy': 7,
  58         'max_segment_size':128*1024 # 1024=KiB
  59     }
  60     def _create_lit(self, cap):
  61         return FakeCHKFileNode(cap)
  62     def _create_immutable(self, cap):
  63         return FakeCHKFileNode(cap)
  64     def _create_mutable(self, cap):
  65         return FakeMutableFileNode(None,
  66                                    None,
  67                                    self.encoding_params, None).init_from_cap(cap)
  68     def create_mutable_file(self, contents="", keysize=None,
  69                             version=SDMF_VERSION):
  70         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  71         return n.create(contents, version=version)
  72
  73 class FakeUploader(service.Service):
  74     name = "uploader"
  75     def upload(self, uploadable):
  76         d = uploadable.get_size()
  77         d.addCallback(lambda size: uploadable.read(size))
  78         def _got_data(datav):
  79             data = "".join(datav)
  80             n = create_chk_filenode(data)
  81             results = upload.UploadResults()
  82             results.uri = n.get_uri()
  83             return results
  84         d.addCallback(_got_data)
  85         return d
  86     def get_helper_info(self):
  87         return (None, False)
  88
  89 class FakeIServer:
  90     def __init__(self, binaryserverid):
  91         self.binaryserverid = binaryserverid
  92     def get_name(self): return "short"
  93     def get_longname(self): return "long"
  94     def get_serverid(self): return self.binaryserverid
  95
  96 def build_one_ds():
  97     ds = DownloadStatus("storage_index", 1234)
  98     now = time.time()
  99
 100     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 101     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 102     storage_index = hashutil.storage_index_hash("SI")
 103     e0 = ds.add_segment_request(0, now)
 104     e0.activate(now+0.5)
 105     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 106     e1 = ds.add_segment_request(1, now+2)
 107     e1.error(now+3)
 108     # two outstanding requests
 109     e2 = ds.add_segment_request(2, now+4)
 110     e3 = ds.add_segment_request(3, now+5)
 111     del e2,e3 # hush pyflakes
 112
 113     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 114     e = ds.add_segment_request(4, now)
 115     e.activate(now)
 116     e.deliver(now, 0, 140, 0.5)
 117
 118     e = ds.add_dyhb_request(serverA, now)
 119     e.finished([1,2], now+1)
 120     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 121
 122     e = ds.add_read_event(0, 120, now)
 123     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 124     e.finished(now+1)
 125     e = ds.add_read_event(120, 30, now+2) # left unfinished
 126
 127     e = ds.add_block_request(serverA, 1, 100, 20, now)
 128     e.finished(20, now+1)
 129     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 130
 131     # make sure that add_read_event() can come first too
 132     ds1 = DownloadStatus(storage_index, 1234)
 133     e = ds1.add_read_event(0, 120, now)
 134     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 135     e.finished(now+1)
 136
 137     return ds
 138
 139 class FakeHistory:
 140     _all_upload_status = [upload.UploadStatus()]
 141     _all_download_status = [build_one_ds()]
 142     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 143     _all_publish_statuses = [publish.PublishStatus()]
 144     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 145
 146     def list_all_upload_statuses(self):
 147         return self._all_upload_status
 148     def list_all_download_statuses(self):
 149         return self._all_download_status
 150     def list_all_mapupdate_statuses(self):
 151         return self._all_mapupdate_statuses
 152     def list_all_publish_statuses(self):
 153         return self._all_publish_statuses
 154     def list_all_retrieve_statuses(self):
 155         return self._all_retrieve_statuses
 156     def list_all_helper_statuses(self):
 157         return []
 158
 159 class FakeClient(Client):
 160     def __init__(self):
 161         # don't upcall to Client.__init__, since we only want to initialize a
 162         # minimal subset
 163         service.MultiService.__init__(self)
 164         self.nodeid = "fake_nodeid"
 165         self.nickname = "fake_nickname"
 166         self.introducer_furl = "None"
 167         self.stats_provider = FakeStatsProvider()
 168         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 169         self.helper = None
 170         self.convergence = "some random string"
 171         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 172         self.introducer_client = None
 173         self.history = FakeHistory()
 174         self.uploader = FakeUploader()
 175         self.uploader.setServiceParent(self)
 176         self.blacklist = None
 177         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 178                                        self.uploader, None,
 179                                        None, None, None)
 180         self.mutable_file_default = SDMF_VERSION
 181
 182     def startService(self):
 183         return service.MultiService.startService(self)
 184     def stopService(self):
 185         return service.MultiService.stopService(self)
 186
 187     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 188
 189 class WebMixin(object):
 190     def setUp(self):
 191         self.s = FakeClient()
 192         self.s.startService()
 193         self.staticdir = self.mktemp()
 194         self.clock = Clock()
 195         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 196                                       clock=self.clock)
 197         self.ws.setServiceParent(self.s)
 198         self.webish_port = self.ws.getPortnum()
 199         self.webish_url = self.ws.getURL()
 200         assert self.webish_url.endswith("/")
 201         self.webish_url = self.webish_url[:-1] # these tests add their own /
 202
 203         l = [ self.s.create_dirnode() for x in range(6) ]
 204         d = defer.DeferredList(l)
 205         def _then(res):
 206             self.public_root = res[0][1]
 207             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 208             self.public_url = "/uri/" + self.public_root.get_uri()
 209             self.private_root = res[1][1]
 210
 211             foo = res[2][1]
 212             self._foo_node = foo
 213             self._foo_uri = foo.get_uri()
 214             self._foo_readonly_uri = foo.get_readonly_uri()
 215             self._foo_verifycap = foo.get_verify_cap().to_string()
 216             # NOTE: we ignore the deferred on all set_uri() calls, because we
 217             # know the fake nodes do these synchronously
 218             self.public_root.set_uri(u"foo", foo.get_uri(),
 219                                      foo.get_readonly_uri())
 220
 221             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 222             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 223             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 224
 225             # sdmf
 226             # XXX: Do we ever use this?
 227             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 228
 229             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 230
 231             # mdmf
 232             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 233             assert self._quux_txt_uri.startswith("URI:MDMF")
 234             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 235
 236             foo.set_uri(u"empty", res[3][1].get_uri(),
 237                         res[3][1].get_readonly_uri())
 238             sub_uri = res[4][1].get_uri()
 239             self._sub_uri = sub_uri
 240             foo.set_uri(u"sub", sub_uri, sub_uri)
 241             sub = self.s.create_node_from_uri(sub_uri)
 242
 243             _ign, n, blocking_uri = self.makefile(1)
 244             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 245
 246             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 247             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 248             # still think of it as an umlaut
 249             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 250
 251             _ign, n, baz_file = self.makefile(2)
 252             self._baz_file_uri = baz_file
 253             sub.set_uri(u"baz.txt", baz_file, baz_file)
 254
 255             _ign, n, self._bad_file_uri = self.makefile(3)
 256             # this uri should not be downloadable
 257             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 258
 259             rodir = res[5][1]
 260             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 261                                      rodir.get_readonly_uri())
 262             rodir.set_uri(u"nor", baz_file, baz_file)
 263
 264             # public/
 265             # public/foo/
 266             # public/foo/bar.txt
 267             # public/foo/baz.txt
 268             # public/foo/quux.txt
 269             # public/foo/blockingfile
 270             # public/foo/empty/
 271             # public/foo/sub/
 272             # public/foo/sub/baz.txt
 273             # public/reedownlee/
 274             # public/reedownlee/nor
 275             self.NEWFILE_CONTENTS = "newfile contents\n"
 276
 277             return foo.get_metadata_for(u"bar.txt")
 278         d.addCallback(_then)
 279         def _got_metadata(metadata):
 280             self._bar_txt_metadata = metadata
 281         d.addCallback(_got_metadata)
 282         return d
 283
 284     def makefile(self, number):
 285         contents = "contents of file %s\n" % number
 286         n = create_chk_filenode(contents)
 287         return contents, n, n.get_uri()
 288
 289     def makefile_mutable(self, number, mdmf=False):
 290         contents = "contents of mutable file %s\n" % number
 291         n = create_mutable_filenode(contents, mdmf)
 292         return contents, n, n.get_uri(), n.get_readonly_uri()
 293
 294     def tearDown(self):
 295         return self.s.stopService()
 296
 297     def failUnlessIsBarDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 299
 300     def failUnlessIsQuuxDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 302
 303     def failUnlessIsBazDotTxt(self, res):
 304         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 305
 306     def failUnlessIsBarJSON(self, res):
 307         data = simplejson.loads(res)
 308         self.failUnless(isinstance(data, list))
 309         self.failUnlessEqual(data[0], "filenode")
 310         self.failUnless(isinstance(data[1], dict))
 311         self.failIf(data[1]["mutable"])
 312         self.failIfIn("rw_uri", data[1]) # immutable
 313         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 314         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 315         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 316
 317     def failUnlessIsQuuxJSON(self, res, readonly=False):
 318         data = simplejson.loads(res)
 319         self.failUnless(isinstance(data, list))
 320         self.failUnlessEqual(data[0], "filenode")
 321         self.failUnless(isinstance(data[1], dict))
 322         metadata = data[1]
 323         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 324
 325     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 326         self.failUnless(metadata['mutable'])
 327         if readonly:
 328             self.failIfIn("rw_uri", metadata)
 329         else:
 330             self.failUnlessIn("rw_uri", metadata)
 331             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 332         self.failUnlessIn("ro_uri", metadata)
 333         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 334         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 335
 336     def failUnlessIsFooJSON(self, res):
 337         data = simplejson.loads(res)
 338         self.failUnless(isinstance(data, list))
 339         self.failUnlessEqual(data[0], "dirnode", res)
 340         self.failUnless(isinstance(data[1], dict))
 341         self.failUnless(data[1]["mutable"])
 342         self.failUnlessIn("rw_uri", data[1]) # mutable
 343         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 344         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 345         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 346
 347         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 348         self.failUnlessEqual(kidnames,
 349                              [u"bar.txt", u"baz.txt", u"blockingfile",
 350                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 351         kids = dict( [(unicode(name),value)
 352                       for (name,value)
 353                       in data[1]["children"].iteritems()] )
 354         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 355         self.failUnlessIn("metadata", kids[u"sub"][1])
 356         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 357         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 358         self.failUnlessIn("linkcrtime", tahoe_md)
 359         self.failUnlessIn("linkmotime", tahoe_md)
 360         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 361         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 362         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 363         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 364                                    self._bar_txt_verifycap)
 365         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 366         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 367         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 368                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 369         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 370                                    self._bar_txt_uri)
 371         self.failUnlessIn("quux.txt", kids)
 372         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 373                                    self._quux_txt_uri)
 374         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 375                                    self._quux_txt_readonly_uri)
 376
 377     def GET(self, urlpath, followRedirect=False, return_response=False,
 378             **kwargs):
 379         # if return_response=True, this fires with (data, statuscode,
 380         # respheaders) instead of just data.
 381         assert not isinstance(urlpath, unicode)
 382         url = self.webish_url + urlpath
 383         factory = HTTPClientGETFactory(url, method="GET",
 384                                        followRedirect=followRedirect, **kwargs)
 385         reactor.connectTCP("localhost", self.webish_port, factory)
 386         d = factory.deferred
 387         def _got_data(data):
 388             return (data, factory.status, factory.response_headers)
 389         if return_response:
 390             d.addCallback(_got_data)
 391         return factory.deferred
 392
 393     def HEAD(self, urlpath, return_response=False, **kwargs):
 394         # this requires some surgery, because twisted.web.client doesn't want
 395         # to give us back the response headers.
 396         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 397         reactor.connectTCP("localhost", self.webish_port, factory)
 398         d = factory.deferred
 399         def _got_data(data):
 400             return (data, factory.status, factory.response_headers)
 401         if return_response:
 402             d.addCallback(_got_data)
 403         return factory.deferred
 404
 405     def PUT(self, urlpath, data, **kwargs):
 406         url = self.webish_url + urlpath
 407         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 408
 409     def DELETE(self, urlpath):
 410         url = self.webish_url + urlpath
 411         return client.getPage(url, method="DELETE")
 412
 413     def POST(self, urlpath, followRedirect=False, **fields):
 414         sepbase = "boogabooga"
 415         sep = "--" + sepbase
 416         form = []
 417         form.append(sep)
 418         form.append('Content-Disposition: form-data; name="_charset"')
 419         form.append('')
 420         form.append('UTF-8')
 421         form.append(sep)
 422         for name, value in fields.iteritems():
 423             if isinstance(value, tuple):
 424                 filename, value = value
 425                 form.append('Content-Disposition: form-data; name="%s"; '
 426                             'filename="%s"' % (name, filename.encode("utf-8")))
 427             else:
 428                 form.append('Content-Disposition: form-data; name="%s"' % name)
 429             form.append('')
 430             if isinstance(value, unicode):
 431                 value = value.encode("utf-8")
 432             else:
 433                 value = str(value)
 434             assert isinstance(value, str)
 435             form.append(value)
 436             form.append(sep)
 437         form[-1] += "--"
 438         body = ""
 439         headers = {}
 440         if fields:
 441             body = "\r\n".join(form) + "\r\n"
 442             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 443         return self.POST2(urlpath, body, headers, followRedirect)
 444
 445     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 446         url = self.webish_url + urlpath
 447         return client.getPage(url, method="POST", postdata=body,
 448                               headers=headers, followRedirect=followRedirect)
 449
 450     def shouldFail(self, res, expected_failure, which,
 451                    substring=None, response_substring=None):
 452         if isinstance(res, failure.Failure):
 453             res.trap(expected_failure)
 454             if substring:
 455                 self.failUnlessIn(substring, str(res), which)
 456             if response_substring:
 457                 self.failUnlessIn(response_substring, res.value.response, which)
 458         else:
 459             self.fail("%s was supposed to raise %s, not get '%s'" %
 460                       (which, expected_failure, res))
 461
 462     def shouldFail2(self, expected_failure, which, substring,
 463                     response_substring,
 464                     callable, *args, **kwargs):
 465         assert substring is None or isinstance(substring, str)
 466         assert response_substring is None or isinstance(response_substring, str)
 467         d = defer.maybeDeferred(callable, *args, **kwargs)
 468         def done(res):
 469             if isinstance(res, failure.Failure):
 470                 res.trap(expected_failure)
 471                 if substring:
 472                     self.failUnlessIn(substring, str(res), which)
 473                 if response_substring:
 474                     self.failUnlessIn(response_substring, res.value.response, which)
 475             else:
 476                 self.fail("%s was supposed to raise %s, not get '%s'" %
 477                           (which, expected_failure, res))
 478         d.addBoth(done)
 479         return d
 480
 481     def should404(self, res, which):
 482         if isinstance(res, failure.Failure):
 483             res.trap(error.Error)
 484             self.failUnlessReallyEqual(res.value.status, "404")
 485         else:
 486             self.fail("%s was supposed to Error(404), not get '%s'" %
 487                       (which, res))
 488
 489     def should302(self, res, which):
 490         if isinstance(res, failure.Failure):
 491             res.trap(error.Error)
 492             self.failUnlessReallyEqual(res.value.status, "302")
 493         else:
 494             self.fail("%s was supposed to Error(302), not get '%s'" %
 495                       (which, res))
 496
 497
 498 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 499     def test_create(self):
 500         pass
 501
 502     def test_welcome(self):
 503         d = self.GET("/")
 504         def _check(res):
 505             self.failUnlessIn('Welcome To Tahoe-LAFS', res)
 506             self.failUnlessIn(FAVICON_MARKUP, res)
 507
 508             self.s.basedir = 'web/test_welcome'
 509             fileutil.make_dirs("web/test_welcome")
 510             fileutil.make_dirs("web/test_welcome/private")
 511             return self.GET("/")
 512         d.addCallback(_check)
 513         return d
 514
 515     def test_status(self):
 516         h = self.s.get_history()
 517         dl_num = h.list_all_download_statuses()[0].get_counter()
 518         ul_num = h.list_all_upload_statuses()[0].get_counter()
 519         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 520         pub_num = h.list_all_publish_statuses()[0].get_counter()
 521         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 522         d = self.GET("/status", followRedirect=True)
 523         def _check(res):
 524             self.failUnlessIn('Upload and Download Status', res)
 525             self.failUnlessIn('"down-%d"' % dl_num, res)
 526             self.failUnlessIn('"up-%d"' % ul_num, res)
 527             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 528             self.failUnlessIn('"publish-%d"' % pub_num, res)
 529             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 530         d.addCallback(_check)
 531         d.addCallback(lambda res: self.GET("/status/?t=json"))
 532         def _check_json(res):
 533             data = simplejson.loads(res)
 534             self.failUnless(isinstance(data, dict))
 535             #active = data["active"]
 536             # TODO: test more. We need a way to fake an active operation
 537             # here.
 538         d.addCallback(_check_json)
 539
 540         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 541         def _check_dl(res):
 542             self.failUnlessIn("File Download Status", res)
 543         d.addCallback(_check_dl)
 544         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 545         def _check_dl_json(res):
 546             data = simplejson.loads(res)
 547             self.failUnless(isinstance(data, dict))
 548             self.failUnlessIn("read", data)
 549             self.failUnlessEqual(data["read"][0]["length"], 120)
 550             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 551             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 552             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 553             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 554             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 555             # serverids[] keys are strings, since that's what JSON does, but
 556             # we'd really like them to be ints
 557             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 558             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 559             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 560             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 561             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 562             self.failUnlessIn("dyhb", data)
 563             self.failUnlessIn("misc", data)
 564         d.addCallback(_check_dl_json)
 565         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 566         def _check_ul(res):
 567             self.failUnlessIn("File Upload Status", res)
 568         d.addCallback(_check_ul)
 569         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 570         def _check_mapupdate(res):
 571             self.failUnlessIn("Mutable File Servermap Update Status", res)
 572         d.addCallback(_check_mapupdate)
 573         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 574         def _check_publish(res):
 575             self.failUnlessIn("Mutable File Publish Status", res)
 576         d.addCallback(_check_publish)
 577         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 578         def _check_retrieve(res):
 579             self.failUnlessIn("Mutable File Retrieve Status", res)
 580         d.addCallback(_check_retrieve)
 581
 582         return d
 583
 584     def test_status_numbers(self):
 585         drrm = status.DownloadResultsRendererMixin()
 586         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 587         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 588         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 589         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 590         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 591         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 592         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 593         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 594         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 595
 596         urrm = status.UploadResultsRendererMixin()
 597         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 598         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 599         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 600         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 601         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 602         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 603         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 604         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 605         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 606
 607     def test_GET_FILEURL(self):
 608         d = self.GET(self.public_url + "/foo/bar.txt")
 609         d.addCallback(self.failUnlessIsBarDotTxt)
 610         return d
 611
 612     def test_GET_FILEURL_range(self):
 613         headers = {"range": "bytes=1-10"}
 614         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 615                      return_response=True)
 616         def _got((res, status, headers)):
 617             self.failUnlessReallyEqual(int(status), 206)
 618             self.failUnless(headers.has_key("content-range"))
 619             self.failUnlessReallyEqual(headers["content-range"][0],
 620                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 621             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 622         d.addCallback(_got)
 623         return d
 624
 625     def test_GET_FILEURL_partial_range(self):
 626         headers = {"range": "bytes=5-"}
 627         length  = len(self.BAR_CONTENTS)
 628         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 629                      return_response=True)
 630         def _got((res, status, headers)):
 631             self.failUnlessReallyEqual(int(status), 206)
 632             self.failUnless(headers.has_key("content-range"))
 633             self.failUnlessReallyEqual(headers["content-range"][0],
 634                                        "bytes 5-%d/%d" % (length-1, length))
 635             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 636         d.addCallback(_got)
 637         return d
 638
 639     def test_GET_FILEURL_partial_end_range(self):
 640         headers = {"range": "bytes=-5"}
 641         length  = len(self.BAR_CONTENTS)
 642         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 643                      return_response=True)
 644         def _got((res, status, headers)):
 645             self.failUnlessReallyEqual(int(status), 206)
 646             self.failUnless(headers.has_key("content-range"))
 647             self.failUnlessReallyEqual(headers["content-range"][0],
 648                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 649             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 650         d.addCallback(_got)
 651         return d
 652
 653     def test_GET_FILEURL_partial_range_overrun(self):
 654         headers = {"range": "bytes=100-200"}
 655         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 656                              "416 Requested Range not satisfiable",
 657                              "First beyond end of file",
 658                              self.GET, self.public_url + "/foo/bar.txt",
 659                              headers=headers)
 660         return d
 661
 662     def test_HEAD_FILEURL_range(self):
 663         headers = {"range": "bytes=1-10"}
 664         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 665                      return_response=True)
 666         def _got((res, status, headers)):
 667             self.failUnlessReallyEqual(res, "")
 668             self.failUnlessReallyEqual(int(status), 206)
 669             self.failUnless(headers.has_key("content-range"))
 670             self.failUnlessReallyEqual(headers["content-range"][0],
 671                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 672         d.addCallback(_got)
 673         return d
 674
 675     def test_HEAD_FILEURL_partial_range(self):
 676         headers = {"range": "bytes=5-"}
 677         length  = len(self.BAR_CONTENTS)
 678         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 679                      return_response=True)
 680         def _got((res, status, headers)):
 681             self.failUnlessReallyEqual(int(status), 206)
 682             self.failUnless(headers.has_key("content-range"))
 683             self.failUnlessReallyEqual(headers["content-range"][0],
 684                                        "bytes 5-%d/%d" % (length-1, length))
 685         d.addCallback(_got)
 686         return d
 687
 688     def test_HEAD_FILEURL_partial_end_range(self):
 689         headers = {"range": "bytes=-5"}
 690         length  = len(self.BAR_CONTENTS)
 691         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 692                      return_response=True)
 693         def _got((res, status, headers)):
 694             self.failUnlessReallyEqual(int(status), 206)
 695             self.failUnless(headers.has_key("content-range"))
 696             self.failUnlessReallyEqual(headers["content-range"][0],
 697                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 698         d.addCallback(_got)
 699         return d
 700
 701     def test_HEAD_FILEURL_partial_range_overrun(self):
 702         headers = {"range": "bytes=100-200"}
 703         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 704                              "416 Requested Range not satisfiable",
 705                              "",
 706                              self.HEAD, self.public_url + "/foo/bar.txt",
 707                              headers=headers)
 708         return d
 709
 710     def test_GET_FILEURL_range_bad(self):
 711         headers = {"range": "BOGUS=fizbop-quarnak"}
 712         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 713                      return_response=True)
 714         def _got((res, status, headers)):
 715             self.failUnlessReallyEqual(int(status), 200)
 716             self.failUnless(not headers.has_key("content-range"))
 717             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 718         d.addCallback(_got)
 719         return d
 720
 721     def test_HEAD_FILEURL(self):
 722         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 723         def _got((res, status, headers)):
 724             self.failUnlessReallyEqual(res, "")
 725             self.failUnlessReallyEqual(headers["content-length"][0],
 726                                        str(len(self.BAR_CONTENTS)))
 727             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 728         d.addCallback(_got)
 729         return d
 730
 731     def test_GET_FILEURL_named(self):
 732         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 733         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 734         d = self.GET(base + "/@@name=/blah.txt")
 735         d.addCallback(self.failUnlessIsBarDotTxt)
 736         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 737         d.addCallback(self.failUnlessIsBarDotTxt)
 738         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 739         d.addCallback(self.failUnlessIsBarDotTxt)
 740         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 741         d.addCallback(self.failUnlessIsBarDotTxt)
 742         save_url = base + "?save=true&filename=blah.txt"
 743         d.addCallback(lambda res: self.GET(save_url))
 744         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 745         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 746         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 747         u_url = base + "?save=true&filename=" + u_fn_e
 748         d.addCallback(lambda res: self.GET(u_url))
 749         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 750         return d
 751
 752     def test_PUT_FILEURL_named_bad(self):
 753         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 754         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 755                              "400 Bad Request",
 756                              "/file can only be used with GET or HEAD",
 757                              self.PUT, base + "/@@name=/blah.txt", "")
 758         return d
 759
 760
 761     def test_GET_DIRURL_named_bad(self):
 762         base = "/file/%s" % urllib.quote(self._foo_uri)
 763         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 764                              "400 Bad Request",
 765                              "is not a file-cap",
 766                              self.GET, base + "/@@name=/blah.txt")
 767         return d
 768
 769     def test_GET_slash_file_bad(self):
 770         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 771                              "404 Not Found",
 772                              "/file must be followed by a file-cap and a name",
 773                              self.GET, "/file")
 774         return d
 775
 776     def test_GET_unhandled_URI_named(self):
 777         contents, n, newuri = self.makefile(12)
 778         verifier_cap = n.get_verify_cap().to_string()
 779         base = "/file/%s" % urllib.quote(verifier_cap)
 780         # client.create_node_from_uri() can't handle verify-caps
 781         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 782                              "400 Bad Request", "is not a file-cap",
 783                              self.GET, base)
 784         return d
 785
 786     def test_GET_unhandled_URI(self):
 787         contents, n, newuri = self.makefile(12)
 788         verifier_cap = n.get_verify_cap().to_string()
 789         base = "/uri/%s" % urllib.quote(verifier_cap)
 790         # client.create_node_from_uri() can't handle verify-caps
 791         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 792                              "400 Bad Request",
 793                              "GET unknown URI type: can only do t=info",
 794                              self.GET, base)
 795         return d
 796
 797     def test_GET_FILE_URI(self):
 798         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 799         d = self.GET(base)
 800         d.addCallback(self.failUnlessIsBarDotTxt)
 801         return d
 802
 803     def test_GET_FILE_URI_mdmf(self):
 804         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 805         d = self.GET(base)
 806         d.addCallback(self.failUnlessIsQuuxDotTxt)
 807         return d
 808
 809     def test_GET_FILE_URI_mdmf_extensions(self):
 810         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 811         d = self.GET(base)
 812         d.addCallback(self.failUnlessIsQuuxDotTxt)
 813         return d
 814
 815     def test_GET_FILE_URI_mdmf_readonly(self):
 816         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 817         d = self.GET(base)
 818         d.addCallback(self.failUnlessIsQuuxDotTxt)
 819         return d
 820
 821     def test_GET_FILE_URI_badchild(self):
 822         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 823         errmsg = "Files have no children, certainly not named 'boguschild'"
 824         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 825                              "400 Bad Request", errmsg,
 826                              self.GET, base)
 827         return d
 828
 829     def test_PUT_FILE_URI_badchild(self):
 830         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 831         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 832         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 833                              "400 Bad Request", errmsg,
 834                              self.PUT, base, "")
 835         return d
 836
 837     def test_PUT_FILE_URI_mdmf(self):
 838         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 839         self._quux_new_contents = "new_contents"
 840         d = self.GET(base)
 841         d.addCallback(lambda res:
 842             self.failUnlessIsQuuxDotTxt(res))
 843         d.addCallback(lambda ignored:
 844             self.PUT(base, self._quux_new_contents))
 845         d.addCallback(lambda ignored:
 846             self.GET(base))
 847         d.addCallback(lambda res:
 848             self.failUnlessReallyEqual(res, self._quux_new_contents))
 849         return d
 850
 851     def test_PUT_FILE_URI_mdmf_extensions(self):
 852         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 853         self._quux_new_contents = "new_contents"
 854         d = self.GET(base)
 855         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 856         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 857         d.addCallback(lambda ignored: self.GET(base))
 858         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 859                                                        res))
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf_readonly(self):
 863         # We're not allowed to PUT things to a readonly cap.
 864         base = "/uri/%s" % self._quux_txt_readonly_uri
 865         d = self.GET(base)
 866         d.addCallback(lambda res:
 867             self.failUnlessIsQuuxDotTxt(res))
 868         # What should we get here? We get a 500 error now; that's not right.
 869         d.addCallback(lambda ignored:
 870             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 871                              "400 Bad Request", "read-only cap",
 872                              self.PUT, base, "new data"))
 873         return d
 874
 875     def test_PUT_FILE_URI_sdmf_readonly(self):
 876         # We're not allowed to put things to a readonly cap.
 877         base = "/uri/%s" % self._baz_txt_readonly_uri
 878         d = self.GET(base)
 879         d.addCallback(lambda res:
 880             self.failUnlessIsBazDotTxt(res))
 881         d.addCallback(lambda ignored:
 882             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 883                              "400 Bad Request", "read-only cap",
 884                              self.PUT, base, "new_data"))
 885         return d
 886
 887     # TODO: version of this with a Unicode filename
 888     def test_GET_FILEURL_save(self):
 889         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 890                      return_response=True)
 891         def _got((res, statuscode, headers)):
 892             content_disposition = headers["content-disposition"][0]
 893             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 894             self.failUnlessIsBarDotTxt(res)
 895         d.addCallback(_got)
 896         return d
 897
 898     def test_GET_FILEURL_missing(self):
 899         d = self.GET(self.public_url + "/foo/missing")
 900         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 901         return d
 902
 903     def test_GET_FILEURL_info_mdmf(self):
 904         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 905         def _got(res):
 906             self.failUnlessIn("mutable file (mdmf)", res)
 907             self.failUnlessIn(self._quux_txt_uri, res)
 908             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 909         d.addCallback(_got)
 910         return d
 911
 912     def test_GET_FILEURL_info_mdmf_readonly(self):
 913         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 914         def _got(res):
 915             self.failUnlessIn("mutable file (mdmf)", res)
 916             self.failIfIn(self._quux_txt_uri, res)
 917             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 918         d.addCallback(_got)
 919         return d
 920
 921     def test_GET_FILEURL_info_sdmf(self):
 922         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 923         def _got(res):
 924             self.failUnlessIn("mutable file (sdmf)", res)
 925             self.failUnlessIn(self._baz_txt_uri, res)
 926         d.addCallback(_got)
 927         return d
 928
 929     def test_GET_FILEURL_info_mdmf_extensions(self):
 930         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 931         def _got(res):
 932             self.failUnlessIn("mutable file (mdmf)", res)
 933             self.failUnlessIn(self._quux_txt_uri, res)
 934             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 935         d.addCallback(_got)
 936         return d
 937
 938     def test_PUT_overwrite_only_files(self):
 939         # create a directory, put a file in that directory.
 940         contents, n, filecap = self.makefile(8)
 941         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 942         d.addCallback(lambda res:
 943             self.PUT(self.public_url + "/foo/dir/file1.txt",
 944                      self.NEWFILE_CONTENTS))
 945         # try to overwrite the file with replace=only-files
 946         # (this should work)
 947         d.addCallback(lambda res:
 948             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 949                      filecap))
 950         d.addCallback(lambda res:
 951             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 952                  "There was already a child by that name, and you asked me "
 953                  "to not replace it",
 954                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 955                  filecap))
 956         return d
 957
 958     def test_PUT_NEWFILEURL(self):
 959         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 960         # TODO: we lose the response code, so we can't check this
 961         #self.failUnlessReallyEqual(responsecode, 201)
 962         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 963         d.addCallback(lambda res:
 964                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 965                                                       self.NEWFILE_CONTENTS))
 966         return d
 967
 968     def test_PUT_NEWFILEURL_not_mutable(self):
 969         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 970                      self.NEWFILE_CONTENTS)
 971         # TODO: we lose the response code, so we can't check this
 972         #self.failUnlessReallyEqual(responsecode, 201)
 973         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 974         d.addCallback(lambda res:
 975                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 976                                                       self.NEWFILE_CONTENTS))
 977         return d
 978
 979     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 980         # this should get us a few segments of an MDMF mutable file,
 981         # which we can then test for.
 982         contents = self.NEWFILE_CONTENTS * 300000
 983         d = self.PUT("/uri?format=mdmf",
 984                      contents)
 985         def _got_filecap(filecap):
 986             self.failUnless(filecap.startswith("URI:MDMF"))
 987             return filecap
 988         d.addCallback(_got_filecap)
 989         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
 990         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
 991         return d
 992
 993     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
 994         contents = self.NEWFILE_CONTENTS * 300000
 995         d = self.PUT("/uri?format=sdmf",
 996                      contents)
 997         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
 998         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
 999         return d
1000
1001     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1002         contents = self.NEWFILE_CONTENTS * 300000
1003         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1004                                     400, "Bad Request", "Unknown format: foo",
1005                                     self.PUT, "/uri?format=foo",
1006                                     contents)
1007
1008     def test_PUT_NEWFILEURL_range_bad(self):
1009         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1010         target = self.public_url + "/foo/new.txt"
1011         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1012                              "501 Not Implemented",
1013                              "Content-Range in PUT not yet supported",
1014                              # (and certainly not for immutable files)
1015                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1016                              headers=headers)
1017         d.addCallback(lambda res:
1018                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1019         return d
1020
1021     def test_PUT_NEWFILEURL_mutable(self):
1022         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1023                      self.NEWFILE_CONTENTS)
1024         # TODO: we lose the response code, so we can't check this
1025         #self.failUnlessReallyEqual(responsecode, 201)
1026         def _check_uri(res):
1027             u = uri.from_string_mutable_filenode(res)
1028             self.failUnless(u.is_mutable())
1029             self.failIf(u.is_readonly())
1030             return res
1031         d.addCallback(_check_uri)
1032         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1033         d.addCallback(lambda res:
1034                       self.failUnlessMutableChildContentsAre(self._foo_node,
1035                                                              u"new.txt",
1036                                                              self.NEWFILE_CONTENTS))
1037         return d
1038
1039     def test_PUT_NEWFILEURL_mutable_toobig(self):
1040         # It is okay to upload large mutable files, so we should be able
1041         # to do that.
1042         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1043                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1044         return d
1045
1046     def test_PUT_NEWFILEURL_replace(self):
1047         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1048         # TODO: we lose the response code, so we can't check this
1049         #self.failUnlessReallyEqual(responsecode, 200)
1050         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1051         d.addCallback(lambda res:
1052                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1053                                                       self.NEWFILE_CONTENTS))
1054         return d
1055
1056     def test_PUT_NEWFILEURL_bad_t(self):
1057         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1058                              "PUT to a file: bad t=bogus",
1059                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1060                              "contents")
1061         return d
1062
1063     def test_PUT_NEWFILEURL_no_replace(self):
1064         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1065                      self.NEWFILE_CONTENTS)
1066         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1067                   "409 Conflict",
1068                   "There was already a child by that name, and you asked me "
1069                   "to not replace it")
1070         return d
1071
1072     def test_PUT_NEWFILEURL_mkdirs(self):
1073         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1074         fn = self._foo_node
1075         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1076         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1077         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1078         d.addCallback(lambda res:
1079                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1080                                                       self.NEWFILE_CONTENTS))
1081         return d
1082
1083     def test_PUT_NEWFILEURL_blocked(self):
1084         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1085                      self.NEWFILE_CONTENTS)
1086         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1087                   "409 Conflict",
1088                   "Unable to create directory 'blockingfile': a file was in the way")
1089         return d
1090
1091     def test_PUT_NEWFILEURL_emptyname(self):
1092         # an empty pathname component (i.e. a double-slash) is disallowed
1093         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1094                              "400 Bad Request",
1095                              "The webapi does not allow empty pathname components",
1096                              self.PUT, self.public_url + "/foo//new.txt", "")
1097         return d
1098
1099     def test_DELETE_FILEURL(self):
1100         d = self.DELETE(self.public_url + "/foo/bar.txt")
1101         d.addCallback(lambda res:
1102                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1103         return d
1104
1105     def test_DELETE_FILEURL_missing(self):
1106         d = self.DELETE(self.public_url + "/foo/missing")
1107         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1108         return d
1109
1110     def test_DELETE_FILEURL_missing2(self):
1111         d = self.DELETE(self.public_url + "/missing/missing")
1112         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1113         return d
1114
1115     def failUnlessHasBarDotTxtMetadata(self, res):
1116         data = simplejson.loads(res)
1117         self.failUnless(isinstance(data, list))
1118         self.failUnlessIn("metadata", data[1])
1119         self.failUnlessIn("tahoe", data[1]["metadata"])
1120         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1121         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1122         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1123                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1124
1125     def test_GET_FILEURL_json(self):
1126         # twisted.web.http.parse_qs ignores any query args without an '=', so
1127         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1128         # instead. This may make it tricky to emulate the S3 interface
1129         # completely.
1130         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1131         def _check1(data):
1132             self.failUnlessIsBarJSON(data)
1133             self.failUnlessHasBarDotTxtMetadata(data)
1134             return
1135         d.addCallback(_check1)
1136         return d
1137
1138     def test_GET_FILEURL_json_mutable_type(self):
1139         # The JSON should include format, which says whether the
1140         # file is SDMF or MDMF
1141         d = self.PUT("/uri?format=mdmf",
1142                      self.NEWFILE_CONTENTS * 300000)
1143         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1144         def _got_json(json, version):
1145             data = simplejson.loads(json)
1146             assert "filenode" == data[0]
1147             data = data[1]
1148             assert isinstance(data, dict)
1149
1150             self.failUnlessIn("format", data)
1151             self.failUnlessEqual(data["format"], version)
1152
1153         d.addCallback(_got_json, "MDMF")
1154         # Now make an SDMF file and check that it is reported correctly.
1155         d.addCallback(lambda ignored:
1156             self.PUT("/uri?format=sdmf",
1157                       self.NEWFILE_CONTENTS * 300000))
1158         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1159         d.addCallback(_got_json, "SDMF")
1160         return d
1161
1162     def test_GET_FILEURL_json_mdmf(self):
1163         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1164         d.addCallback(self.failUnlessIsQuuxJSON)
1165         return d
1166
1167     def test_GET_FILEURL_json_missing(self):
1168         d = self.GET(self.public_url + "/foo/missing?json")
1169         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1170         return d
1171
1172     def test_GET_FILEURL_uri(self):
1173         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1174         def _check(res):
1175             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1176         d.addCallback(_check)
1177         d.addCallback(lambda res:
1178                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1179         def _check2(res):
1180             # for now, for files, uris and readonly-uris are the same
1181             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1182         d.addCallback(_check2)
1183         return d
1184
1185     def test_GET_FILEURL_badtype(self):
1186         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1187                                  "bad t=bogus",
1188                                  self.GET,
1189                                  self.public_url + "/foo/bar.txt?t=bogus")
1190         return d
1191
1192     def test_CSS_FILE(self):
1193         d = self.GET("/tahoe.css", followRedirect=True)
1194         def _check(res):
1195             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1196             self.failUnless(CSS_STYLE.search(res), res)
1197         d.addCallback(_check)
1198         return d
1199
1200     def test_GET_FILEURL_uri_missing(self):
1201         d = self.GET(self.public_url + "/foo/missing?t=uri")
1202         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1203         return d
1204
1205     def _check_upload_and_mkdir_forms(self, html):
1206         # We should have a form to create a file, with radio buttons that allow
1207         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1208         self.failUnlessIn('name="t" value="upload"', html)
1209         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1210         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1211         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1212
1213         # We should also have the ability to create a mutable directory, with
1214         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1215         # or MDMF directory.
1216         self.failUnlessIn('name="t" value="mkdir"', html)
1217         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1218         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1219
1220         self.failUnlessIn(FAVICON_MARKUP, html)
1221
1222     def test_GET_DIRECTORY_html(self):
1223         d = self.GET(self.public_url + "/foo", followRedirect=True)
1224         def _check(html):
1225             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1226             self._check_upload_and_mkdir_forms(html)
1227             self.failUnlessIn("quux", html)
1228         d.addCallback(_check)
1229         return d
1230
1231     def test_GET_root_html(self):
1232         d = self.GET("/")
1233         d.addCallback(self._check_upload_and_mkdir_forms)
1234         return d
1235
1236     def test_GET_DIRURL(self):
1237         # the addSlash means we get a redirect here
1238         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1239         ROOT = "../../.."
1240         d = self.GET(self.public_url + "/foo", followRedirect=True)
1241         def _check(res):
1242             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1243
1244             # the FILE reference points to a URI, but it should end in bar.txt
1245             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1246                        (ROOT, urllib.quote(self._bar_txt_uri)))
1247             get_bar = "".join([r'<td>FILE</td>',
1248                                r'\s+<td>',
1249                                r'<a href="%s">bar.txt</a>' % bar_url,
1250                                r'</td>',
1251                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1252                                ])
1253             self.failUnless(re.search(get_bar, res), res)
1254             for label in ['unlink', 'rename']:
1255                 for line in res.split("\n"):
1256                     # find the line that contains the relevant button for bar.txt
1257                     if ("form action" in line and
1258                         ('value="%s"' % (label,)) in line and
1259                         'value="bar.txt"' in line):
1260                         # the form target should use a relative URL
1261                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1262                         self.failUnlessIn('action="%s"' % foo_url, line)
1263                         # and the when_done= should too
1264                         #done_url = urllib.quote(???)
1265                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1266
1267                         # 'unlink' needs to use POST because it directly has a side effect
1268                         if label == 'unlink':
1269                             self.failUnlessIn('method="post"', line)
1270                         break
1271                 else:
1272                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1273
1274             # the DIR reference just points to a URI
1275             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1276             get_sub = ((r'<td>DIR</td>')
1277                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1278             self.failUnless(re.search(get_sub, res), res)
1279         d.addCallback(_check)
1280
1281         # look at a readonly directory
1282         d.addCallback(lambda res:
1283                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1284         def _check2(res):
1285             self.failUnlessIn("(read-only)", res)
1286             self.failIfIn("Upload a file", res)
1287         d.addCallback(_check2)
1288
1289         # and at a directory that contains a readonly directory
1290         d.addCallback(lambda res:
1291                       self.GET(self.public_url, followRedirect=True))
1292         def _check3(res):
1293             self.failUnless(re.search('<td>DIR-RO</td>'
1294                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1295         d.addCallback(_check3)
1296
1297         # and an empty directory
1298         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1299         def _check4(res):
1300             self.failUnlessIn("directory is empty", res)
1301             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1302             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1303         d.addCallback(_check4)
1304
1305         # and at a literal directory
1306         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1307         d.addCallback(lambda res:
1308                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1309         def _check5(res):
1310             self.failUnlessIn('(immutable)', res)
1311             self.failUnless(re.search('<td>FILE</td>'
1312                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1313         d.addCallback(_check5)
1314         return d
1315
1316     def test_GET_DIRURL_badtype(self):
1317         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1318                                  400, "Bad Request",
1319                                  "bad t=bogus",
1320                                  self.GET,
1321                                  self.public_url + "/foo?t=bogus")
1322         return d
1323
1324     def test_GET_DIRURL_json(self):
1325         d = self.GET(self.public_url + "/foo?t=json")
1326         d.addCallback(self.failUnlessIsFooJSON)
1327         return d
1328
1329     def test_GET_DIRURL_json_format(self):
1330         d = self.PUT(self.public_url + \
1331                      "/foo/sdmf.txt?format=sdmf",
1332                      self.NEWFILE_CONTENTS * 300000)
1333         d.addCallback(lambda ignored:
1334             self.PUT(self.public_url + \
1335                      "/foo/mdmf.txt?format=mdmf",
1336                      self.NEWFILE_CONTENTS * 300000))
1337         # Now we have an MDMF and SDMF file in the directory. If we GET
1338         # its JSON, we should see their encodings.
1339         d.addCallback(lambda ignored:
1340             self.GET(self.public_url + "/foo?t=json"))
1341         def _got_json(json):
1342             data = simplejson.loads(json)
1343             assert data[0] == "dirnode"
1344
1345             data = data[1]
1346             kids = data['children']
1347
1348             mdmf_data = kids['mdmf.txt'][1]
1349             self.failUnlessIn("format", mdmf_data)
1350             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1351
1352             sdmf_data = kids['sdmf.txt'][1]
1353             self.failUnlessIn("format", sdmf_data)
1354             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1355         d.addCallback(_got_json)
1356         return d
1357
1358
1359     def test_POST_DIRURL_manifest_no_ophandle(self):
1360         d = self.shouldFail2(error.Error,
1361                              "test_POST_DIRURL_manifest_no_ophandle",
1362                              "400 Bad Request",
1363                              "slow operation requires ophandle=",
1364                              self.POST, self.public_url, t="start-manifest")
1365         return d
1366
1367     def test_POST_DIRURL_manifest(self):
1368         d = defer.succeed(None)
1369         def getman(ignored, output):
1370             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1371                           followRedirect=True)
1372             d.addCallback(self.wait_for_operation, "125")
1373             d.addCallback(self.get_operation_results, "125", output)
1374             return d
1375         d.addCallback(getman, None)
1376         def _got_html(manifest):
1377             self.failUnlessIn("Manifest of SI=", manifest)
1378             self.failUnlessIn("<td>sub</td>", manifest)
1379             self.failUnlessIn(self._sub_uri, manifest)
1380             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1381             self.failUnlessIn(FAVICON_MARKUP, manifest)
1382         d.addCallback(_got_html)
1383
1384         # both t=status and unadorned GET should be identical
1385         d.addCallback(lambda res: self.GET("/operations/125"))
1386         d.addCallback(_got_html)
1387
1388         d.addCallback(getman, "html")
1389         d.addCallback(_got_html)
1390         d.addCallback(getman, "text")
1391         def _got_text(manifest):
1392             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1393             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1394         d.addCallback(_got_text)
1395         d.addCallback(getman, "JSON")
1396         def _got_json(res):
1397             data = res["manifest"]
1398             got = {}
1399             for (path_list, cap) in data:
1400                 got[tuple(path_list)] = cap
1401             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1402             self.failUnlessIn((u"sub", u"baz.txt"), got)
1403             self.failUnlessIn("finished", res)
1404             self.failUnlessIn("origin", res)
1405             self.failUnlessIn("storage-index", res)
1406             self.failUnlessIn("verifycaps", res)
1407             self.failUnlessIn("stats", res)
1408         d.addCallback(_got_json)
1409         return d
1410
1411     def test_POST_DIRURL_deepsize_no_ophandle(self):
1412         d = self.shouldFail2(error.Error,
1413                              "test_POST_DIRURL_deepsize_no_ophandle",
1414                              "400 Bad Request",
1415                              "slow operation requires ophandle=",
1416                              self.POST, self.public_url, t="start-deep-size")
1417         return d
1418
1419     def test_POST_DIRURL_deepsize(self):
1420         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1421                       followRedirect=True)
1422         d.addCallback(self.wait_for_operation, "126")
1423         d.addCallback(self.get_operation_results, "126", "json")
1424         def _got_json(data):
1425             self.failUnlessReallyEqual(data["finished"], True)
1426             size = data["size"]
1427             self.failUnless(size > 1000)
1428         d.addCallback(_got_json)
1429         d.addCallback(self.get_operation_results, "126", "text")
1430         def _got_text(res):
1431             mo = re.search(r'^size: (\d+)$', res, re.M)
1432             self.failUnless(mo, res)
1433             size = int(mo.group(1))
1434             # with directories, the size varies.
1435             self.failUnless(size > 1000)
1436         d.addCallback(_got_text)
1437         return d
1438
1439     def test_POST_DIRURL_deepstats_no_ophandle(self):
1440         d = self.shouldFail2(error.Error,
1441                              "test_POST_DIRURL_deepstats_no_ophandle",
1442                              "400 Bad Request",
1443                              "slow operation requires ophandle=",
1444                              self.POST, self.public_url, t="start-deep-stats")
1445         return d
1446
1447     def test_POST_DIRURL_deepstats(self):
1448         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1449                       followRedirect=True)
1450         d.addCallback(self.wait_for_operation, "127")
1451         d.addCallback(self.get_operation_results, "127", "json")
1452         def _got_json(stats):
1453             expected = {"count-immutable-files": 3,
1454                         "count-mutable-files": 2,
1455                         "count-literal-files": 0,
1456                         "count-files": 5,
1457                         "count-directories": 3,
1458                         "size-immutable-files": 57,
1459                         "size-literal-files": 0,
1460                         #"size-directories": 1912, # varies
1461                         #"largest-directory": 1590,
1462                         "largest-directory-children": 7,
1463                         "largest-immutable-file": 19,
1464                         }
1465             for k,v in expected.iteritems():
1466                 self.failUnlessReallyEqual(stats[k], v,
1467                                            "stats[%s] was %s, not %s" %
1468                                            (k, stats[k], v))
1469             self.failUnlessReallyEqual(stats["size-files-histogram"],
1470                                        [ [11, 31, 3] ])
1471         d.addCallback(_got_json)
1472         return d
1473
1474     def test_POST_DIRURL_stream_manifest(self):
1475         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1476         def _check(res):
1477             self.failUnless(res.endswith("\n"))
1478             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1479             self.failUnlessReallyEqual(len(units), 9)
1480             self.failUnlessEqual(units[-1]["type"], "stats")
1481             first = units[0]
1482             self.failUnlessEqual(first["path"], [])
1483             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1484             self.failUnlessEqual(first["type"], "directory")
1485             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1486             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1487             self.failIfEqual(baz["storage-index"], None)
1488             self.failIfEqual(baz["verifycap"], None)
1489             self.failIfEqual(baz["repaircap"], None)
1490             # XXX: Add quux and baz to this test.
1491             return
1492         d.addCallback(_check)
1493         return d
1494
1495     def test_GET_DIRURL_uri(self):
1496         d = self.GET(self.public_url + "/foo?t=uri")
1497         def _check(res):
1498             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1499         d.addCallback(_check)
1500         return d
1501
1502     def test_GET_DIRURL_readonly_uri(self):
1503         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1504         def _check(res):
1505             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1506         d.addCallback(_check)
1507         return d
1508
1509     def test_PUT_NEWDIRURL(self):
1510         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1511         d.addCallback(lambda res:
1512                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1513         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1514         d.addCallback(self.failUnlessNodeKeysAre, [])
1515         return d
1516
1517     def test_PUT_NEWDIRURL_mdmf(self):
1518         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1519         d.addCallback(lambda res:
1520                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1521         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1522         d.addCallback(lambda node:
1523             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1524         return d
1525
1526     def test_PUT_NEWDIRURL_sdmf(self):
1527         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1528                      "")
1529         d.addCallback(lambda res:
1530                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1531         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1532         d.addCallback(lambda node:
1533             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1534         return d
1535
1536     def test_PUT_NEWDIRURL_bad_format(self):
1537         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1538                                     400, "Bad Request", "Unknown format: foo",
1539                                     self.PUT, self.public_url +
1540                                     "/foo/newdir=?t=mkdir&format=foo", "")
1541
1542     def test_POST_NEWDIRURL(self):
1543         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1544         d.addCallback(lambda res:
1545                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1546         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1547         d.addCallback(self.failUnlessNodeKeysAre, [])
1548         return d
1549
1550     def test_POST_NEWDIRURL_mdmf(self):
1551         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1552         d.addCallback(lambda res:
1553                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1554         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1555         d.addCallback(lambda node:
1556             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1557         return d
1558
1559     def test_POST_NEWDIRURL_sdmf(self):
1560         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1561         d.addCallback(lambda res:
1562             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1563         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1564         d.addCallback(lambda node:
1565             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1566         return d
1567
1568     def test_POST_NEWDIRURL_bad_format(self):
1569         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1570                                     400, "Bad Request", "Unknown format: foo",
1571                                     self.POST2, self.public_url + \
1572                                     "/foo/newdir?t=mkdir&format=foo", "")
1573
1574     def test_POST_NEWDIRURL_emptyname(self):
1575         # an empty pathname component (i.e. a double-slash) is disallowed
1576         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1577                              "400 Bad Request",
1578                              "The webapi does not allow empty pathname components, i.e. a double slash",
1579                              self.POST, self.public_url + "//?t=mkdir")
1580         return d
1581
1582     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1583         (newkids, caps) = self._create_initial_children()
1584         query = "/foo/newdir?t=mkdir-with-children"
1585         if version == MDMF_VERSION:
1586             query += "&format=mdmf"
1587         elif version == SDMF_VERSION:
1588             query += "&format=sdmf"
1589         else:
1590             version = SDMF_VERSION # for later
1591         d = self.POST2(self.public_url + query,
1592                        simplejson.dumps(newkids))
1593         def _check(uri):
1594             n = self.s.create_node_from_uri(uri.strip())
1595             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1596             self.failUnlessEqual(n._node.get_version(), version)
1597             d2.addCallback(lambda ign:
1598                            self.failUnlessROChildURIIs(n, u"child-imm",
1599                                                        caps['filecap1']))
1600             d2.addCallback(lambda ign:
1601                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1602                                                        caps['filecap2']))
1603             d2.addCallback(lambda ign:
1604                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1605                                                        caps['filecap3']))
1606             d2.addCallback(lambda ign:
1607                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1608                                                        caps['unknown_rocap']))
1609             d2.addCallback(lambda ign:
1610                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1611                                                        caps['unknown_rwcap']))
1612             d2.addCallback(lambda ign:
1613                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1614                                                        caps['unknown_immcap']))
1615             d2.addCallback(lambda ign:
1616                            self.failUnlessRWChildURIIs(n, u"dirchild",
1617                                                        caps['dircap']))
1618             d2.addCallback(lambda ign:
1619                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1620                                                        caps['litdircap']))
1621             d2.addCallback(lambda ign:
1622                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1623                                                        caps['emptydircap']))
1624             return d2
1625         d.addCallback(_check)
1626         d.addCallback(lambda res:
1627                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1629         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1630         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1631         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1632         return d
1633
1634     def test_POST_NEWDIRURL_initial_children(self):
1635         return self._do_POST_NEWDIRURL_initial_children_test()
1636
1637     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1638         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1639
1640     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1641         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1642
1643     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1644         (newkids, caps) = self._create_initial_children()
1645         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1646                                     400, "Bad Request", "Unknown format: foo",
1647                                     self.POST2, self.public_url + \
1648                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1649                                     simplejson.dumps(newkids))
1650
1651     def test_POST_NEWDIRURL_immutable(self):
1652         (newkids, caps) = self._create_immutable_children()
1653         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1654                        simplejson.dumps(newkids))
1655         def _check(uri):
1656             n = self.s.create_node_from_uri(uri.strip())
1657             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1658             d2.addCallback(lambda ign:
1659                            self.failUnlessROChildURIIs(n, u"child-imm",
1660                                                        caps['filecap1']))
1661             d2.addCallback(lambda ign:
1662                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1663                                                        caps['unknown_immcap']))
1664             d2.addCallback(lambda ign:
1665                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1666                                                        caps['immdircap']))
1667             d2.addCallback(lambda ign:
1668                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1669                                                        caps['litdircap']))
1670             d2.addCallback(lambda ign:
1671                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1672                                                        caps['emptydircap']))
1673             return d2
1674         d.addCallback(_check)
1675         d.addCallback(lambda res:
1676                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1677         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1678         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1679         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1681         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1682         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1683         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1684         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1685         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1686         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1687         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1688         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1689         d.addErrback(self.explain_web_error)
1690         return d
1691
1692     def test_POST_NEWDIRURL_immutable_bad(self):
1693         (newkids, caps) = self._create_initial_children()
1694         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1695                              "400 Bad Request",
1696                              "needed to be immutable but was not",
1697                              self.POST2,
1698                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1699                              simplejson.dumps(newkids))
1700         return d
1701
1702     def test_PUT_NEWDIRURL_exists(self):
1703         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1704         d.addCallback(lambda res:
1705                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1706         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1707         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1708         return d
1709
1710     def test_PUT_NEWDIRURL_blocked(self):
1711         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1712                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1713                              self.PUT,
1714                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1717         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_mkdir_p(self):
1722         d = defer.succeed(None)
1723         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1724         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1725         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1726         def mkdir_p(mkpnode):
1727             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1728             d = self.POST(url)
1729             def made_subsub(ssuri):
1730                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1731                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1732                 d = self.POST(url)
1733                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1734                 return d
1735             d.addCallback(made_subsub)
1736             return d
1737         d.addCallback(mkdir_p)
1738         return d
1739
1740     def test_PUT_NEWDIRURL_mkdirs(self):
1741         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1742         d.addCallback(lambda res:
1743                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res:
1745                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1746         d.addCallback(lambda res:
1747                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1748         d.addCallback(self.failUnlessNodeKeysAre, [])
1749         return d
1750
1751     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1752         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1753         d.addCallback(lambda ignored:
1754             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1755         d.addCallback(lambda ignored:
1756             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1757         d.addCallback(lambda ignored:
1758             self._foo_node.get_child_at_path(u"subdir"))
1759         def _got_subdir(subdir):
1760             # XXX: What we want?
1761             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1762             self.failUnlessNodeHasChild(subdir, u"newdir")
1763             return subdir.get_child_at_path(u"newdir")
1764         d.addCallback(_got_subdir)
1765         d.addCallback(lambda newdir:
1766             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1767         return d
1768
1769     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1770         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1771         d.addCallback(lambda ignored:
1772             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1773         d.addCallback(lambda ignored:
1774             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1775         d.addCallback(lambda ignored:
1776             self._foo_node.get_child_at_path(u"subdir"))
1777         def _got_subdir(subdir):
1778             # XXX: What we want?
1779             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1780             self.failUnlessNodeHasChild(subdir, u"newdir")
1781             return subdir.get_child_at_path(u"newdir")
1782         d.addCallback(_got_subdir)
1783         d.addCallback(lambda newdir:
1784             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1785         return d
1786
1787     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1788         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1789                                     400, "Bad Request", "Unknown format: foo",
1790                                     self.PUT, self.public_url + \
1791                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1792                                     "")
1793
1794     def test_DELETE_DIRURL(self):
1795         d = self.DELETE(self.public_url + "/foo")
1796         d.addCallback(lambda res:
1797                       self.failIfNodeHasChild(self.public_root, u"foo"))
1798         return d
1799
1800     def test_DELETE_DIRURL_missing(self):
1801         d = self.DELETE(self.public_url + "/foo/missing")
1802         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1803         d.addCallback(lambda res:
1804                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1805         return d
1806
1807     def test_DELETE_DIRURL_missing2(self):
1808         d = self.DELETE(self.public_url + "/missing")
1809         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1810         return d
1811
1812     def dump_root(self):
1813         print "NODEWALK"
1814         w = webish.DirnodeWalkerMixin()
1815         def visitor(childpath, childnode, metadata):
1816             print childpath
1817         d = w.walk(self.public_root, visitor)
1818         return d
1819
1820     def failUnlessNodeKeysAre(self, node, expected_keys):
1821         for k in expected_keys:
1822             assert isinstance(k, unicode)
1823         d = node.list()
1824         def _check(children):
1825             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1826         d.addCallback(_check)
1827         return d
1828     def failUnlessNodeHasChild(self, node, name):
1829         assert isinstance(name, unicode)
1830         d = node.list()
1831         def _check(children):
1832             self.failUnlessIn(name, children)
1833         d.addCallback(_check)
1834         return d
1835     def failIfNodeHasChild(self, node, name):
1836         assert isinstance(name, unicode)
1837         d = node.list()
1838         def _check(children):
1839             self.failIfIn(name, children)
1840         d.addCallback(_check)
1841         return d
1842
1843     def failUnlessChildContentsAre(self, node, name, expected_contents):
1844         assert isinstance(name, unicode)
1845         d = node.get_child_at_path(name)
1846         d.addCallback(lambda node: download_to_data(node))
1847         def _check(contents):
1848             self.failUnlessReallyEqual(contents, expected_contents)
1849         d.addCallback(_check)
1850         return d
1851
1852     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1853         assert isinstance(name, unicode)
1854         d = node.get_child_at_path(name)
1855         d.addCallback(lambda node: node.download_best_version())
1856         def _check(contents):
1857             self.failUnlessReallyEqual(contents, expected_contents)
1858         d.addCallback(_check)
1859         return d
1860
1861     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1862         assert isinstance(name, unicode)
1863         d = node.get_child_at_path(name)
1864         def _check(child):
1865             self.failUnless(child.is_unknown() or not child.is_readonly())
1866             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1867             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1868             expected_ro_uri = self._make_readonly(expected_uri)
1869             if expected_ro_uri:
1870                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1871         d.addCallback(_check)
1872         return d
1873
1874     def failUnlessROChildURIIs(self, node, name, expected_uri):
1875         assert isinstance(name, unicode)
1876         d = node.get_child_at_path(name)
1877         def _check(child):
1878             self.failUnless(child.is_unknown() or child.is_readonly())
1879             self.failUnlessReallyEqual(child.get_write_uri(), None)
1880             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1881             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1882         d.addCallback(_check)
1883         return d
1884
1885     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1886         assert isinstance(name, unicode)
1887         d = node.get_child_at_path(name)
1888         def _check(child):
1889             self.failUnless(child.is_unknown() or not child.is_readonly())
1890             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1891             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1892             expected_ro_uri = self._make_readonly(got_uri)
1893             if expected_ro_uri:
1894                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1895         d.addCallback(_check)
1896         return d
1897
1898     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1899         assert isinstance(name, unicode)
1900         d = node.get_child_at_path(name)
1901         def _check(child):
1902             self.failUnless(child.is_unknown() or child.is_readonly())
1903             self.failUnlessReallyEqual(child.get_write_uri(), None)
1904             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1905             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1906         d.addCallback(_check)
1907         return d
1908
1909     def failUnlessCHKURIHasContents(self, got_uri, contents):
1910         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1911
1912     def test_POST_upload(self):
1913         d = self.POST(self.public_url + "/foo", t="upload",
1914                       file=("new.txt", self.NEWFILE_CONTENTS))
1915         fn = self._foo_node
1916         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1917         d.addCallback(lambda res:
1918                       self.failUnlessChildContentsAre(fn, u"new.txt",
1919                                                       self.NEWFILE_CONTENTS))
1920         return d
1921
1922     def test_POST_upload_unicode(self):
1923         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1924         d = self.POST(self.public_url + "/foo", t="upload",
1925                       file=(filename, self.NEWFILE_CONTENTS))
1926         fn = self._foo_node
1927         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1928         d.addCallback(lambda res:
1929                       self.failUnlessChildContentsAre(fn, filename,
1930                                                       self.NEWFILE_CONTENTS))
1931         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1932         d.addCallback(lambda res: self.GET(target_url))
1933         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1934                                                                   self.NEWFILE_CONTENTS,
1935                                                                   contents))
1936         return d
1937
1938     def test_POST_upload_unicode_named(self):
1939         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1940         d = self.POST(self.public_url + "/foo", t="upload",
1941                       name=filename,
1942                       file=("overridden", self.NEWFILE_CONTENTS))
1943         fn = self._foo_node
1944         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1945         d.addCallback(lambda res:
1946                       self.failUnlessChildContentsAre(fn, filename,
1947                                                       self.NEWFILE_CONTENTS))
1948         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1949         d.addCallback(lambda res: self.GET(target_url))
1950         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1951                                                                   self.NEWFILE_CONTENTS,
1952                                                                   contents))
1953         return d
1954
1955     def test_POST_upload_no_link(self):
1956         d = self.POST("/uri", t="upload",
1957                       file=("new.txt", self.NEWFILE_CONTENTS))
1958         def _check_upload_results(page):
1959             # this should be a page which describes the results of the upload
1960             # that just finished.
1961             self.failUnlessIn("Upload Results:", page)
1962             self.failUnlessIn("URI:", page)
1963             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1964             mo = uri_re.search(page)
1965             self.failUnless(mo, page)
1966             new_uri = mo.group(1)
1967             return new_uri
1968         d.addCallback(_check_upload_results)
1969         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1970         return d
1971
1972     def test_POST_upload_no_link_whendone(self):
1973         d = self.POST("/uri", t="upload", when_done="/",
1974                       file=("new.txt", self.NEWFILE_CONTENTS))
1975         d.addBoth(self.shouldRedirect, "/")
1976         return d
1977
1978     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1979         d = defer.maybeDeferred(callable, *args, **kwargs)
1980         def done(res):
1981             if isinstance(res, failure.Failure):
1982                 res.trap(error.PageRedirect)
1983                 statuscode = res.value.status
1984                 target = res.value.location
1985                 return checker(statuscode, target)
1986             self.fail("%s: callable was supposed to redirect, not return '%s'"
1987                       % (which, res))
1988         d.addBoth(done)
1989         return d
1990
1991     def test_POST_upload_no_link_whendone_results(self):
1992         def check(statuscode, target):
1993             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1994             self.failUnless(target.startswith(self.webish_url), target)
1995             return client.getPage(target, method="GET")
1996         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1997                                  check,
1998                                  self.POST, "/uri", t="upload",
1999                                  when_done="/uri/%(uri)s",
2000                                  file=("new.txt", self.NEWFILE_CONTENTS))
2001         d.addCallback(lambda res:
2002                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2003         return d
2004
2005     def test_POST_upload_no_link_mutable(self):
2006         d = self.POST("/uri", t="upload", mutable="true",
2007                       file=("new.txt", self.NEWFILE_CONTENTS))
2008         def _check(filecap):
2009             filecap = filecap.strip()
2010             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2011             self.filecap = filecap
2012             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2013             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2014             n = self.s.create_node_from_uri(filecap)
2015             return n.download_best_version()
2016         d.addCallback(_check)
2017         def _check2(data):
2018             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2019             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2020         d.addCallback(_check2)
2021         def _check3(data):
2022             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2023             return self.GET("/file/%s" % urllib.quote(self.filecap))
2024         d.addCallback(_check3)
2025         def _check4(data):
2026             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2027         d.addCallback(_check4)
2028         return d
2029
2030     def test_POST_upload_no_link_mutable_toobig(self):
2031         # The SDMF size limit is no longer in place, so we should be
2032         # able to upload mutable files that are as large as we want them
2033         # to be.
2034         d = self.POST("/uri", t="upload", mutable="true",
2035                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2036         return d
2037
2038
2039     def test_POST_upload_format_unlinked(self):
2040         def _check_upload_unlinked(ign, format, uri_prefix):
2041             filename = format + ".txt"
2042             d = self.POST("/uri?t=upload&format=" + format,
2043                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2044             def _got_results(results):
2045                 if format.upper() in ("SDMF", "MDMF"):
2046                     # webapi.rst says this returns a filecap
2047                     filecap = results
2048                 else:
2049                     # for immutable, it returns an "upload results page", and
2050                     # the filecap is buried inside
2051                     line = [l for l in results.split("\n") if "URI: " in l][0]
2052                     mo = re.search(r'<span>([^<]+)</span>', line)
2053                     filecap = mo.group(1)
2054                 self.failUnless(filecap.startswith(uri_prefix),
2055                                 (uri_prefix, filecap))
2056                 return self.GET("/uri/%s?t=json" % filecap)
2057             d.addCallback(_got_results)
2058             def _got_json(json):
2059                 data = simplejson.loads(json)
2060                 data = data[1]
2061                 self.failUnlessIn("format", data)
2062                 self.failUnlessEqual(data["format"], format.upper())
2063             d.addCallback(_got_json)
2064             return d
2065         d = defer.succeed(None)
2066         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2067         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2068         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2069         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2070         return d
2071
2072     def test_POST_upload_bad_format_unlinked(self):
2073         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2074                                     400, "Bad Request", "Unknown format: foo",
2075                                     self.POST,
2076                                     "/uri?t=upload&format=foo",
2077                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2078
2079     def test_POST_upload_format(self):
2080         def _check_upload(ign, format, uri_prefix, fn=None):
2081             filename = format + ".txt"
2082             d = self.POST(self.public_url +
2083                           "/foo?t=upload&format=" + format,
2084                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2085             def _got_filecap(filecap):
2086                 if fn is not None:
2087                     filenameu = unicode(filename)
2088                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2089                 self.failUnless(filecap.startswith(uri_prefix))
2090                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2091             d.addCallback(_got_filecap)
2092             def _got_json(json):
2093                 data = simplejson.loads(json)
2094                 data = data[1]
2095                 self.failUnlessIn("format", data)
2096                 self.failUnlessEqual(data["format"], format.upper())
2097             d.addCallback(_got_json)
2098             return d
2099
2100         d = defer.succeed(None)
2101         d.addCallback(_check_upload, "chk", "URI:CHK")
2102         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2103         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2104         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2105         return d
2106
2107     def test_POST_upload_bad_format(self):
2108         return self.shouldHTTPError("POST_upload_bad_format",
2109                                     400, "Bad Request", "Unknown format: foo",
2110                                     self.POST, self.public_url + \
2111                                     "/foo?t=upload&format=foo",
2112                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2113
2114     def test_POST_upload_mutable(self):
2115         # this creates a mutable file
2116         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2117                       file=("new.txt", self.NEWFILE_CONTENTS))
2118         fn = self._foo_node
2119         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2120         d.addCallback(lambda res:
2121                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2122                                                              self.NEWFILE_CONTENTS))
2123         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2124         def _got(newnode):
2125             self.failUnless(IMutableFileNode.providedBy(newnode))
2126             self.failUnless(newnode.is_mutable())
2127             self.failIf(newnode.is_readonly())
2128             self._mutable_node = newnode
2129             self._mutable_uri = newnode.get_uri()
2130         d.addCallback(_got)
2131
2132         # now upload it again and make sure that the URI doesn't change
2133         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2134         d.addCallback(lambda res:
2135                       self.POST(self.public_url + "/foo", t="upload",
2136                                 mutable="true",
2137                                 file=("new.txt", NEWER_CONTENTS)))
2138         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2139         d.addCallback(lambda res:
2140                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2141                                                              NEWER_CONTENTS))
2142         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2143         def _got2(newnode):
2144             self.failUnless(IMutableFileNode.providedBy(newnode))
2145             self.failUnless(newnode.is_mutable())
2146             self.failIf(newnode.is_readonly())
2147             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2148         d.addCallback(_got2)
2149
2150         # upload a second time, using PUT instead of POST
2151         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2152         d.addCallback(lambda res:
2153                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2154         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2155         d.addCallback(lambda res:
2156                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2157                                                              NEW2_CONTENTS))
2158
2159         # finally list the directory, since mutable files are displayed
2160         # slightly differently
2161
2162         d.addCallback(lambda res:
2163                       self.GET(self.public_url + "/foo/",
2164                                followRedirect=True))
2165         def _check_page(res):
2166             # TODO: assert more about the contents
2167             self.failUnlessIn("SSK", res)
2168             return res
2169         d.addCallback(_check_page)
2170
2171         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2172         def _got3(newnode):
2173             self.failUnless(IMutableFileNode.providedBy(newnode))
2174             self.failUnless(newnode.is_mutable())
2175             self.failIf(newnode.is_readonly())
2176             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2177         d.addCallback(_got3)
2178
2179         # look at the JSON form of the enclosing directory
2180         d.addCallback(lambda res:
2181                       self.GET(self.public_url + "/foo/?t=json",
2182                                followRedirect=True))
2183         def _check_page_json(res):
2184             parsed = simplejson.loads(res)
2185             self.failUnlessEqual(parsed[0], "dirnode")
2186             children = dict( [(unicode(name),value)
2187                               for (name,value)
2188                               in parsed[1]["children"].iteritems()] )
2189             self.failUnlessIn(u"new.txt", children)
2190             new_json = children[u"new.txt"]
2191             self.failUnlessEqual(new_json[0], "filenode")
2192             self.failUnless(new_json[1]["mutable"])
2193             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2194             ro_uri = self._mutable_node.get_readonly().to_string()
2195             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2196         d.addCallback(_check_page_json)
2197
2198         # and the JSON form of the file
2199         d.addCallback(lambda res:
2200                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2201         def _check_file_json(res):
2202             parsed = simplejson.loads(res)
2203             self.failUnlessEqual(parsed[0], "filenode")
2204             self.failUnless(parsed[1]["mutable"])
2205             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2206             ro_uri = self._mutable_node.get_readonly().to_string()
2207             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2208         d.addCallback(_check_file_json)
2209
2210         # and look at t=uri and t=readonly-uri
2211         d.addCallback(lambda res:
2212                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2213         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2214         d.addCallback(lambda res:
2215                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2216         def _check_ro_uri(res):
2217             ro_uri = self._mutable_node.get_readonly().to_string()
2218             self.failUnlessReallyEqual(res, ro_uri)
2219         d.addCallback(_check_ro_uri)
2220
2221         # make sure we can get to it from /uri/URI
2222         d.addCallback(lambda res:
2223                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2224         d.addCallback(lambda res:
2225                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2226
2227         # and that HEAD computes the size correctly
2228         d.addCallback(lambda res:
2229                       self.HEAD(self.public_url + "/foo/new.txt",
2230                                 return_response=True))
2231         def _got_headers((res, status, headers)):
2232             self.failUnlessReallyEqual(res, "")
2233             self.failUnlessReallyEqual(headers["content-length"][0],
2234                                        str(len(NEW2_CONTENTS)))
2235             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2236         d.addCallback(_got_headers)
2237
2238         # make sure that outdated size limits aren't enforced anymore.
2239         d.addCallback(lambda ignored:
2240             self.POST(self.public_url + "/foo", t="upload",
2241                       mutable="true",
2242                       file=("new.txt",
2243                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2244         d.addErrback(self.dump_error)
2245         return d
2246
2247     def test_POST_upload_mutable_toobig(self):
2248         # SDMF had a size limti that was removed a while ago. MDMF has
2249         # never had a size limit. Test to make sure that we do not
2250         # encounter errors when trying to upload large mutable files,
2251         # since there should be no coded prohibitions regarding large
2252         # mutable files.
2253         d = self.POST(self.public_url + "/foo",
2254                       t="upload", mutable="true",
2255                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2256         return d
2257
2258     def dump_error(self, f):
2259         # if the web server returns an error code (like 400 Bad Request),
2260         # web.client.getPage puts the HTTP response body into the .response
2261         # attribute of the exception object that it gives back. It does not
2262         # appear in the Failure's repr(), so the ERROR that trial displays
2263         # will be rather terse and unhelpful. addErrback this method to the
2264         # end of your chain to get more information out of these errors.
2265         if f.check(error.Error):
2266             print "web.error.Error:"
2267             print f
2268             print f.value.response
2269         return f
2270
2271     def test_POST_upload_replace(self):
2272         d = self.POST(self.public_url + "/foo", t="upload",
2273                       file=("bar.txt", self.NEWFILE_CONTENTS))
2274         fn = self._foo_node
2275         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2276         d.addCallback(lambda res:
2277                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2278                                                       self.NEWFILE_CONTENTS))
2279         return d
2280
2281     def test_POST_upload_no_replace_ok(self):
2282         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2283                       file=("new.txt", self.NEWFILE_CONTENTS))
2284         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2285         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2286                                                              self.NEWFILE_CONTENTS))
2287         return d
2288
2289     def test_POST_upload_no_replace_queryarg(self):
2290         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2291                       file=("bar.txt", self.NEWFILE_CONTENTS))
2292         d.addBoth(self.shouldFail, error.Error,
2293                   "POST_upload_no_replace_queryarg",
2294                   "409 Conflict",
2295                   "There was already a child by that name, and you asked me "
2296                   "to not replace it")
2297         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2298         d.addCallback(self.failUnlessIsBarDotTxt)
2299         return d
2300
2301     def test_POST_upload_no_replace_field(self):
2302         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2303                       file=("bar.txt", self.NEWFILE_CONTENTS))
2304         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2305                   "409 Conflict",
2306                   "There was already a child by that name, and you asked me "
2307                   "to not replace it")
2308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2309         d.addCallback(self.failUnlessIsBarDotTxt)
2310         return d
2311
2312     def test_POST_upload_whendone(self):
2313         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2314                       file=("new.txt", self.NEWFILE_CONTENTS))
2315         d.addBoth(self.shouldRedirect, "/THERE")
2316         fn = self._foo_node
2317         d.addCallback(lambda res:
2318                       self.failUnlessChildContentsAre(fn, u"new.txt",
2319                                                       self.NEWFILE_CONTENTS))
2320         return d
2321
2322     def test_POST_upload_named(self):
2323         fn = self._foo_node
2324         d = self.POST(self.public_url + "/foo", t="upload",
2325                       name="new.txt", file=self.NEWFILE_CONTENTS)
2326         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2327         d.addCallback(lambda res:
2328                       self.failUnlessChildContentsAre(fn, u"new.txt",
2329                                                       self.NEWFILE_CONTENTS))
2330         return d
2331
2332     def test_POST_upload_named_badfilename(self):
2333         d = self.POST(self.public_url + "/foo", t="upload",
2334                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2335         d.addBoth(self.shouldFail, error.Error,
2336                   "test_POST_upload_named_badfilename",
2337                   "400 Bad Request",
2338                   "name= may not contain a slash",
2339                   )
2340         # make sure that nothing was added
2341         d.addCallback(lambda res:
2342                       self.failUnlessNodeKeysAre(self._foo_node,
2343                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2344                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2345                                                   u"sub"]))
2346         return d
2347
2348     def test_POST_FILEURL_check(self):
2349         bar_url = self.public_url + "/foo/bar.txt"
2350         d = self.POST(bar_url, t="check")
2351         def _check(res):
2352             self.failUnlessIn("Healthy :", res)
2353         d.addCallback(_check)
2354         redir_url = "http://allmydata.org/TARGET"
2355         def _check2(statuscode, target):
2356             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2357             self.failUnlessReallyEqual(target, redir_url)
2358         d.addCallback(lambda res:
2359                       self.shouldRedirect2("test_POST_FILEURL_check",
2360                                            _check2,
2361                                            self.POST, bar_url,
2362                                            t="check",
2363                                            when_done=redir_url))
2364         d.addCallback(lambda res:
2365                       self.POST(bar_url, t="check", return_to=redir_url))
2366         def _check3(res):
2367             self.failUnlessIn("Healthy :", res)
2368             self.failUnlessIn("Return to file", res)
2369             self.failUnlessIn(redir_url, res)
2370         d.addCallback(_check3)
2371
2372         d.addCallback(lambda res:
2373                       self.POST(bar_url, t="check", output="JSON"))
2374         def _check_json(res):
2375             data = simplejson.loads(res)
2376             self.failUnlessIn("storage-index", data)
2377             self.failUnless(data["results"]["healthy"])
2378         d.addCallback(_check_json)
2379
2380         return d
2381
2382     def test_POST_FILEURL_check_and_repair(self):
2383         bar_url = self.public_url + "/foo/bar.txt"
2384         d = self.POST(bar_url, t="check", repair="true")
2385         def _check(res):
2386             self.failUnlessIn("Healthy :", res)
2387         d.addCallback(_check)
2388         redir_url = "http://allmydata.org/TARGET"
2389         def _check2(statuscode, target):
2390             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2391             self.failUnlessReallyEqual(target, redir_url)
2392         d.addCallback(lambda res:
2393                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2394                                            _check2,
2395                                            self.POST, bar_url,
2396                                            t="check", repair="true",
2397                                            when_done=redir_url))
2398         d.addCallback(lambda res:
2399                       self.POST(bar_url, t="check", return_to=redir_url))
2400         def _check3(res):
2401             self.failUnlessIn("Healthy :", res)
2402             self.failUnlessIn("Return to file", res)
2403             self.failUnlessIn(redir_url, res)
2404         d.addCallback(_check3)
2405         return d
2406
2407     def test_POST_DIRURL_check(self):
2408         foo_url = self.public_url + "/foo/"
2409         d = self.POST(foo_url, t="check")
2410         def _check(res):
2411             self.failUnlessIn("Healthy :", res)
2412         d.addCallback(_check)
2413         redir_url = "http://allmydata.org/TARGET"
2414         def _check2(statuscode, target):
2415             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2416             self.failUnlessReallyEqual(target, redir_url)
2417         d.addCallback(lambda res:
2418                       self.shouldRedirect2("test_POST_DIRURL_check",
2419                                            _check2,
2420                                            self.POST, foo_url,
2421                                            t="check",
2422                                            when_done=redir_url))
2423         d.addCallback(lambda res:
2424                       self.POST(foo_url, t="check", return_to=redir_url))
2425         def _check3(res):
2426             self.failUnlessIn("Healthy :", res)
2427             self.failUnlessIn("Return to file/directory", res)
2428             self.failUnlessIn(redir_url, res)
2429         d.addCallback(_check3)
2430
2431         d.addCallback(lambda res:
2432                       self.POST(foo_url, t="check", output="JSON"))
2433         def _check_json(res):
2434             data = simplejson.loads(res)
2435             self.failUnlessIn("storage-index", data)
2436             self.failUnless(data["results"]["healthy"])
2437         d.addCallback(_check_json)
2438
2439         return d
2440
2441     def test_POST_DIRURL_check_and_repair(self):
2442         foo_url = self.public_url + "/foo/"
2443         d = self.POST(foo_url, t="check", repair="true")
2444         def _check(res):
2445             self.failUnlessIn("Healthy :", res)
2446         d.addCallback(_check)
2447         redir_url = "http://allmydata.org/TARGET"
2448         def _check2(statuscode, target):
2449             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2450             self.failUnlessReallyEqual(target, redir_url)
2451         d.addCallback(lambda res:
2452                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2453                                            _check2,
2454                                            self.POST, foo_url,
2455                                            t="check", repair="true",
2456                                            when_done=redir_url))
2457         d.addCallback(lambda res:
2458                       self.POST(foo_url, t="check", return_to=redir_url))
2459         def _check3(res):
2460             self.failUnlessIn("Healthy :", res)
2461             self.failUnlessIn("Return to file/directory", res)
2462             self.failUnlessIn(redir_url, res)
2463         d.addCallback(_check3)
2464         return d
2465
2466     def test_POST_FILEURL_mdmf_check(self):
2467         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2468         d = self.POST(quux_url, t="check")
2469         def _check(res):
2470             self.failUnlessIn("Healthy", res)
2471         d.addCallback(_check)
2472         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2473         d.addCallback(lambda ignored:
2474                       self.POST(quux_extension_url, t="check"))
2475         d.addCallback(_check)
2476         return d
2477
2478     def test_POST_FILEURL_mdmf_check_and_repair(self):
2479         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2480         d = self.POST(quux_url, t="check", repair="true")
2481         def _check(res):
2482             self.failUnlessIn("Healthy", res)
2483         d.addCallback(_check)
2484         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2485         d.addCallback(lambda ignored:
2486                       self.POST(quux_extension_url, t="check", repair="true"))
2487         d.addCallback(_check)
2488         return d
2489
2490     def wait_for_operation(self, ignored, ophandle):
2491         url = "/operations/" + ophandle
2492         url += "?t=status&output=JSON"
2493         d = self.GET(url)
2494         def _got(res):
2495             data = simplejson.loads(res)
2496             if not data["finished"]:
2497                 d = self.stall(delay=1.0)
2498                 d.addCallback(self.wait_for_operation, ophandle)
2499                 return d
2500             return data
2501         d.addCallback(_got)
2502         return d
2503
2504     def get_operation_results(self, ignored, ophandle, output=None):
2505         url = "/operations/" + ophandle
2506         url += "?t=status"
2507         if output:
2508             url += "&output=" + output
2509         d = self.GET(url)
2510         def _got(res):
2511             if output and output.lower() == "json":
2512                 return simplejson.loads(res)
2513             return res
2514         d.addCallback(_got)
2515         return d
2516
2517     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2518         d = self.shouldFail2(error.Error,
2519                              "test_POST_DIRURL_deepcheck_no_ophandle",
2520                              "400 Bad Request",
2521                              "slow operation requires ophandle=",
2522                              self.POST, self.public_url, t="start-deep-check")
2523         return d
2524
2525     def test_POST_DIRURL_deepcheck(self):
2526         def _check_redirect(statuscode, target):
2527             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2528             self.failUnless(target.endswith("/operations/123"))
2529         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2530                                  self.POST, self.public_url,
2531                                  t="start-deep-check", ophandle="123")
2532         d.addCallback(self.wait_for_operation, "123")
2533         def _check_json(data):
2534             self.failUnlessReallyEqual(data["finished"], True)
2535             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2536             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2537         d.addCallback(_check_json)
2538         d.addCallback(self.get_operation_results, "123", "html")
2539         def _check_html(res):
2540             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2541             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2542             self.failUnlessIn(FAVICON_MARKUP, res)
2543         d.addCallback(_check_html)
2544
2545         d.addCallback(lambda res:
2546                       self.GET("/operations/123/"))
2547         d.addCallback(_check_html) # should be the same as without the slash
2548
2549         d.addCallback(lambda res:
2550                       self.shouldFail2(error.Error, "one", "404 Not Found",
2551                                        "No detailed results for SI bogus",
2552                                        self.GET, "/operations/123/bogus"))
2553
2554         foo_si = self._foo_node.get_storage_index()
2555         foo_si_s = base32.b2a(foo_si)
2556         d.addCallback(lambda res:
2557                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2558         def _check_foo_json(res):
2559             data = simplejson.loads(res)
2560             self.failUnlessEqual(data["storage-index"], foo_si_s)
2561             self.failUnless(data["results"]["healthy"])
2562         d.addCallback(_check_foo_json)
2563         return d
2564
2565     def test_POST_DIRURL_deepcheck_and_repair(self):
2566         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2567                       ophandle="124", output="json", followRedirect=True)
2568         d.addCallback(self.wait_for_operation, "124")
2569         def _check_json(data):
2570             self.failUnlessReallyEqual(data["finished"], True)
2571             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2572             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2573             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2574             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2575             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2576             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2577             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2578             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2579             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2580             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2581         d.addCallback(_check_json)
2582         d.addCallback(self.get_operation_results, "124", "html")
2583         def _check_html(res):
2584             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2585
2586             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2587             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2588             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2589
2590             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2591             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2592             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2593
2594             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2595             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2596             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2597
2598             self.failUnlessIn(FAVICON_MARKUP, res)
2599         d.addCallback(_check_html)
2600         return d
2601
2602     def test_POST_FILEURL_bad_t(self):
2603         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2604                              "POST to file: bad t=bogus",
2605                              self.POST, self.public_url + "/foo/bar.txt",
2606                              t="bogus")
2607         return d
2608
2609     def test_POST_mkdir(self): # return value?
2610         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2611         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2612         d.addCallback(self.failUnlessNodeKeysAre, [])
2613         return d
2614
2615     def test_POST_mkdir_mdmf(self):
2616         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2617         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2618         d.addCallback(lambda node:
2619             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2620         return d
2621
2622     def test_POST_mkdir_sdmf(self):
2623         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2624         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2625         d.addCallback(lambda node:
2626             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2627         return d
2628
2629     def test_POST_mkdir_bad_format(self):
2630         return self.shouldHTTPError("POST_mkdir_bad_format",
2631                                     400, "Bad Request", "Unknown format: foo",
2632                                     self.POST, self.public_url +
2633                                     "/foo?t=mkdir&name=newdir&format=foo")
2634
2635     def test_POST_mkdir_initial_children(self):
2636         (newkids, caps) = self._create_initial_children()
2637         d = self.POST2(self.public_url +
2638                        "/foo?t=mkdir-with-children&name=newdir",
2639                        simplejson.dumps(newkids))
2640         d.addCallback(lambda res:
2641                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2642         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2643         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2644         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2645         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2646         return d
2647
2648     def test_POST_mkdir_initial_children_mdmf(self):
2649         (newkids, caps) = self._create_initial_children()
2650         d = self.POST2(self.public_url +
2651                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2652                        simplejson.dumps(newkids))
2653         d.addCallback(lambda res:
2654                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(lambda node:
2657             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2658         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2659         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2660                        caps['filecap1'])
2661         return d
2662
2663     # XXX: Duplication.
2664     def test_POST_mkdir_initial_children_sdmf(self):
2665         (newkids, caps) = self._create_initial_children()
2666         d = self.POST2(self.public_url +
2667                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2668                        simplejson.dumps(newkids))
2669         d.addCallback(lambda res:
2670                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2671         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672         d.addCallback(lambda node:
2673             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2675         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2676                        caps['filecap1'])
2677         return d
2678
2679     def test_POST_mkdir_initial_children_bad_format(self):
2680         (newkids, caps) = self._create_initial_children()
2681         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2682                                     400, "Bad Request", "Unknown format: foo",
2683                                     self.POST, self.public_url + \
2684                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2685                                     simplejson.dumps(newkids))
2686
2687     def test_POST_mkdir_immutable(self):
2688         (newkids, caps) = self._create_immutable_children()
2689         d = self.POST2(self.public_url +
2690                        "/foo?t=mkdir-immutable&name=newdir",
2691                        simplejson.dumps(newkids))
2692         d.addCallback(lambda res:
2693                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2695         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2696         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2697         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2699         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2700         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2701         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2702         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2703         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2704         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2705         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2706         return d
2707
2708     def test_POST_mkdir_immutable_bad(self):
2709         (newkids, caps) = self._create_initial_children()
2710         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2711                              "400 Bad Request",
2712                              "needed to be immutable but was not",
2713                              self.POST2,
2714                              self.public_url +
2715                              "/foo?t=mkdir-immutable&name=newdir",
2716                              simplejson.dumps(newkids))
2717         return d
2718
2719     def test_POST_mkdir_2(self):
2720         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2721         d.addCallback(lambda res:
2722                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724         d.addCallback(self.failUnlessNodeKeysAre, [])
2725         return d
2726
2727     def test_POST_mkdirs_2(self):
2728         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2729         d.addCallback(lambda res:
2730                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2731         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2732         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2733         d.addCallback(self.failUnlessNodeKeysAre, [])
2734         return d
2735
2736     def test_POST_mkdir_no_parentdir_noredirect(self):
2737         d = self.POST("/uri?t=mkdir")
2738         def _after_mkdir(res):
2739             uri.DirectoryURI.init_from_string(res)
2740         d.addCallback(_after_mkdir)
2741         return d
2742
2743     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2744         d = self.POST("/uri?t=mkdir&format=mdmf")
2745         def _after_mkdir(res):
2746             u = uri.from_string(res)
2747             # Check that this is an MDMF writecap
2748             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2749         d.addCallback(_after_mkdir)
2750         return d
2751
2752     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2753         d = self.POST("/uri?t=mkdir&format=sdmf")
2754         def _after_mkdir(res):
2755             u = uri.from_string(res)
2756             self.failUnlessIsInstance(u, uri.DirectoryURI)
2757         d.addCallback(_after_mkdir)
2758         return d
2759
2760     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2761         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2762                                     400, "Bad Request", "Unknown format: foo",
2763                                     self.POST, self.public_url +
2764                                     "/uri?t=mkdir&format=foo")
2765
2766     def test_POST_mkdir_no_parentdir_noredirect2(self):
2767         # make sure form-based arguments (as on the welcome page) still work
2768         d = self.POST("/uri", t="mkdir")
2769         def _after_mkdir(res):
2770             uri.DirectoryURI.init_from_string(res)
2771         d.addCallback(_after_mkdir)
2772         d.addErrback(self.explain_web_error)
2773         return d
2774
2775     def test_POST_mkdir_no_parentdir_redirect(self):
2776         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2777         d.addBoth(self.shouldRedirect, None, statuscode='303')
2778         def _check_target(target):
2779             target = urllib.unquote(target)
2780             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2781         d.addCallback(_check_target)
2782         return d
2783
2784     def test_POST_mkdir_no_parentdir_redirect2(self):
2785         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2786         d.addBoth(self.shouldRedirect, None, statuscode='303')
2787         def _check_target(target):
2788             target = urllib.unquote(target)
2789             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2790         d.addCallback(_check_target)
2791         d.addErrback(self.explain_web_error)
2792         return d
2793
2794     def _make_readonly(self, u):
2795         ro_uri = uri.from_string(u).get_readonly()
2796         if ro_uri is None:
2797             return None
2798         return ro_uri.to_string()
2799
2800     def _create_initial_children(self):
2801         contents, n, filecap1 = self.makefile(12)
2802         md1 = {"metakey1": "metavalue1"}
2803         filecap2 = make_mutable_file_uri()
2804         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2805         filecap3 = node3.get_readonly_uri()
2806         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2807         dircap = DirectoryNode(node4, None, None).get_uri()
2808         mdmfcap = make_mutable_file_uri(mdmf=True)
2809         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2810         emptydircap = "URI:DIR2-LIT:"
2811         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2812                                                       "ro_uri": self._make_readonly(filecap1),
2813                                                       "metadata": md1, }],
2814                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2815                                                       "ro_uri": self._make_readonly(filecap2)}],
2816                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2817                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2818                                                       "ro_uri": unknown_rocap}],
2819                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2820                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2821                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2822                                                       "ro_uri": self._make_readonly(dircap)}],
2823                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2824                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2825                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2826                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2827                    }
2828         return newkids, {'filecap1': filecap1,
2829                          'filecap2': filecap2,
2830                          'filecap3': filecap3,
2831                          'unknown_rwcap': unknown_rwcap,
2832                          'unknown_rocap': unknown_rocap,
2833                          'unknown_immcap': unknown_immcap,
2834                          'dircap': dircap,
2835                          'litdircap': litdircap,
2836                          'emptydircap': emptydircap,
2837                          'mdmfcap': mdmfcap}
2838
2839     def _create_immutable_children(self):
2840         contents, n, filecap1 = self.makefile(12)
2841         md1 = {"metakey1": "metavalue1"}
2842         tnode = create_chk_filenode("immutable directory contents\n"*10)
2843         dnode = DirectoryNode(tnode, None, None)
2844         assert not dnode.is_mutable()
2845         immdircap = dnode.get_uri()
2846         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2847         emptydircap = "URI:DIR2-LIT:"
2848         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2849                                                       "metadata": md1, }],
2850                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2851                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2852                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2853                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2854                    }
2855         return newkids, {'filecap1': filecap1,
2856                          'unknown_immcap': unknown_immcap,
2857                          'immdircap': immdircap,
2858                          'litdircap': litdircap,
2859                          'emptydircap': emptydircap}
2860
2861     def test_POST_mkdir_no_parentdir_initial_children(self):
2862         (newkids, caps) = self._create_initial_children()
2863         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2864         def _after_mkdir(res):
2865             self.failUnless(res.startswith("URI:DIR"), res)
2866             n = self.s.create_node_from_uri(res)
2867             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2868             d2.addCallback(lambda ign:
2869                            self.failUnlessROChildURIIs(n, u"child-imm",
2870                                                        caps['filecap1']))
2871             d2.addCallback(lambda ign:
2872                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2873                                                        caps['filecap2']))
2874             d2.addCallback(lambda ign:
2875                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2876                                                        caps['filecap3']))
2877             d2.addCallback(lambda ign:
2878                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2879                                                        caps['unknown_rwcap']))
2880             d2.addCallback(lambda ign:
2881                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2882                                                        caps['unknown_rocap']))
2883             d2.addCallback(lambda ign:
2884                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2885                                                        caps['unknown_immcap']))
2886             d2.addCallback(lambda ign:
2887                            self.failUnlessRWChildURIIs(n, u"dirchild",
2888                                                        caps['dircap']))
2889             return d2
2890         d.addCallback(_after_mkdir)
2891         return d
2892
2893     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2894         # the regular /uri?t=mkdir operation is specified to ignore its body.
2895         # Only t=mkdir-with-children pays attention to it.
2896         (newkids, caps) = self._create_initial_children()
2897         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2898                                  400, "Bad Request",
2899                                  "t=mkdir does not accept children=, "
2900                                  "try t=mkdir-with-children instead",
2901                                  self.POST2, "/uri?t=mkdir", # without children
2902                                  simplejson.dumps(newkids))
2903         return d
2904
2905     def test_POST_noparent_bad(self):
2906         d = self.shouldHTTPError("POST_noparent_bad",
2907                                  400, "Bad Request",
2908                                  "/uri accepts only PUT, PUT?t=mkdir, "
2909                                  "POST?t=upload, and POST?t=mkdir",
2910                                  self.POST, "/uri?t=bogus")
2911         return d
2912
2913     def test_POST_mkdir_no_parentdir_immutable(self):
2914         (newkids, caps) = self._create_immutable_children()
2915         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2916         def _after_mkdir(res):
2917             self.failUnless(res.startswith("URI:DIR"), res)
2918             n = self.s.create_node_from_uri(res)
2919             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2920             d2.addCallback(lambda ign:
2921                            self.failUnlessROChildURIIs(n, u"child-imm",
2922                                                           caps['filecap1']))
2923             d2.addCallback(lambda ign:
2924                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2925                                                           caps['unknown_immcap']))
2926             d2.addCallback(lambda ign:
2927                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2928                                                           caps['immdircap']))
2929             d2.addCallback(lambda ign:
2930                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2931                                                           caps['litdircap']))
2932             d2.addCallback(lambda ign:
2933                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2934                                                           caps['emptydircap']))
2935             return d2
2936         d.addCallback(_after_mkdir)
2937         return d
2938
2939     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2940         (newkids, caps) = self._create_initial_children()
2941         d = self.shouldFail2(error.Error,
2942                              "test_POST_mkdir_no_parentdir_immutable_bad",
2943                              "400 Bad Request",
2944                              "needed to be immutable but was not",
2945                              self.POST2,
2946                              "/uri?t=mkdir-immutable",
2947                              simplejson.dumps(newkids))
2948         return d
2949
2950     def test_welcome_page_mkdir_button(self):
2951         # Fetch the welcome page.
2952         d = self.GET("/")
2953         def _after_get_welcome_page(res):
2954             MKDIR_BUTTON_RE = re.compile(
2955                 '<form action="([^"]*)" method="post".*?'
2956                 '<input type="hidden" name="t" value="([^"]*)" />'
2957                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2958                 '<input type="submit" value="Create a directory" />',
2959                 re.I)
2960             mo = MKDIR_BUTTON_RE.search(res)
2961             formaction = mo.group(1)
2962             formt = mo.group(2)
2963             formaname = mo.group(3)
2964             formavalue = mo.group(4)
2965             return (formaction, formt, formaname, formavalue)
2966         d.addCallback(_after_get_welcome_page)
2967         def _after_parse_form(res):
2968             (formaction, formt, formaname, formavalue) = res
2969             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2970         d.addCallback(_after_parse_form)
2971         d.addBoth(self.shouldRedirect, None, statuscode='303')
2972         return d
2973
2974     def test_POST_mkdir_replace(self): # return value?
2975         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2976         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2977         d.addCallback(self.failUnlessNodeKeysAre, [])
2978         return d
2979
2980     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2981         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2982         d.addBoth(self.shouldFail, error.Error,
2983                   "POST_mkdir_no_replace_queryarg",
2984                   "409 Conflict",
2985                   "There was already a child by that name, and you asked me "
2986                   "to not replace it")
2987         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2988         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2989         return d
2990
2991     def test_POST_mkdir_no_replace_field(self): # return value?
2992         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2993                       replace="false")
2994         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2995                   "409 Conflict",
2996                   "There was already a child by that name, and you asked me "
2997                   "to not replace it")
2998         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2999         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3000         return d
3001
3002     def test_POST_mkdir_whendone_field(self):
3003         d = self.POST(self.public_url + "/foo",
3004                       t="mkdir", name="newdir", when_done="/THERE")
3005         d.addBoth(self.shouldRedirect, "/THERE")
3006         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3007         d.addCallback(self.failUnlessNodeKeysAre, [])
3008         return d
3009
3010     def test_POST_mkdir_whendone_queryarg(self):
3011         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3012                       t="mkdir", name="newdir")
3013         d.addBoth(self.shouldRedirect, "/THERE")
3014         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3015         d.addCallback(self.failUnlessNodeKeysAre, [])
3016         return d
3017
3018     def test_POST_bad_t(self):
3019         d = self.shouldFail2(error.Error, "POST_bad_t",
3020                              "400 Bad Request",
3021                              "POST to a directory with bad t=BOGUS",
3022                              self.POST, self.public_url + "/foo", t="BOGUS")
3023         return d
3024
3025     def test_POST_set_children(self, command_name="set_children"):
3026         contents9, n9, newuri9 = self.makefile(9)
3027         contents10, n10, newuri10 = self.makefile(10)
3028         contents11, n11, newuri11 = self.makefile(11)
3029
3030         reqbody = """{
3031                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3032                                                 "size": 0,
3033                                                 "metadata": {
3034                                                   "ctime": 1002777696.7564139,
3035                                                   "mtime": 1002777696.7564139
3036                                                  }
3037                                                } ],
3038                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3039                                                 "size": 1,
3040                                                 "metadata": {
3041                                                   "ctime": 1002777696.7564139,
3042                                                   "mtime": 1002777696.7564139
3043                                                  }
3044                                                } ],
3045                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3046                                                 "size": 2,
3047                                                 "metadata": {
3048                                                   "ctime": 1002777696.7564139,
3049                                                   "mtime": 1002777696.7564139
3050                                                  }
3051                                                } ]
3052                     }""" % (newuri9, newuri10, newuri11)
3053
3054         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3055
3056         d = client.getPage(url, method="POST", postdata=reqbody)
3057         def _then(res):
3058             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3059             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3060             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3061
3062         d.addCallback(_then)
3063         d.addErrback(self.dump_error)
3064         return d
3065
3066     def test_POST_set_children_with_hyphen(self):
3067         return self.test_POST_set_children(command_name="set-children")
3068
3069     def test_POST_link_uri(self):
3070         contents, n, newuri = self.makefile(8)
3071         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3072         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3073         d.addCallback(lambda res:
3074                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3075                                                       contents))
3076         return d
3077
3078     def test_POST_link_uri_replace(self):
3079         contents, n, newuri = self.makefile(8)
3080         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3081         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3082         d.addCallback(lambda res:
3083                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3084                                                       contents))
3085         return d
3086
3087     def test_POST_link_uri_unknown_bad(self):
3088         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3089         d.addBoth(self.shouldFail, error.Error,
3090                   "POST_link_uri_unknown_bad",
3091                   "400 Bad Request",
3092                   "unknown cap in a write slot")
3093         return d
3094
3095     def test_POST_link_uri_unknown_ro_good(self):
3096         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3097         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3098         return d
3099
3100     def test_POST_link_uri_unknown_imm_good(self):
3101         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3102         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3103         return d
3104
3105     def test_POST_link_uri_no_replace_queryarg(self):
3106         contents, n, newuri = self.makefile(8)
3107         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3108                       name="bar.txt", uri=newuri)
3109         d.addBoth(self.shouldFail, error.Error,
3110                   "POST_link_uri_no_replace_queryarg",
3111                   "409 Conflict",
3112                   "There was already a child by that name, and you asked me "
3113                   "to not replace it")
3114         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3115         d.addCallback(self.failUnlessIsBarDotTxt)
3116         return d
3117
3118     def test_POST_link_uri_no_replace_field(self):
3119         contents, n, newuri = self.makefile(8)
3120         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3121                       name="bar.txt", uri=newuri)
3122         d.addBoth(self.shouldFail, error.Error,
3123                   "POST_link_uri_no_replace_field",
3124                   "409 Conflict",
3125                   "There was already a child by that name, and you asked me "
3126                   "to not replace it")
3127         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3128         d.addCallback(self.failUnlessIsBarDotTxt)
3129         return d
3130
3131     def test_POST_delete(self, command_name='delete'):
3132         d = self._foo_node.list()
3133         def _check_before(children):
3134             self.failUnlessIn(u"bar.txt", children)
3135         d.addCallback(_check_before)
3136         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3137         d.addCallback(lambda res: self._foo_node.list())
3138         def _check_after(children):
3139             self.failIfIn(u"bar.txt", children)
3140         d.addCallback(_check_after)
3141         return d
3142
3143     def test_POST_unlink(self):
3144         return self.test_POST_delete(command_name='unlink')
3145
3146     def test_POST_rename_file(self):
3147         d = self.POST(self.public_url + "/foo", t="rename",
3148                       from_name="bar.txt", to_name='wibble.txt')
3149         d.addCallback(lambda res:
3150                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3151         d.addCallback(lambda res:
3152                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3153         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3154         d.addCallback(self.failUnlessIsBarDotTxt)
3155         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3156         d.addCallback(self.failUnlessIsBarJSON)
3157         return d
3158
3159     def test_POST_rename_file_redundant(self):
3160         d = self.POST(self.public_url + "/foo", t="rename",
3161                       from_name="bar.txt", to_name='bar.txt')
3162         d.addCallback(lambda res:
3163                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3167         d.addCallback(self.failUnlessIsBarJSON)
3168         return d
3169
3170     def test_POST_rename_file_replace(self):
3171         # rename a file and replace a directory with it
3172         d = self.POST(self.public_url + "/foo", t="rename",
3173                       from_name="bar.txt", to_name='empty')
3174         d.addCallback(lambda res:
3175                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3176         d.addCallback(lambda res:
3177                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3178         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3179         d.addCallback(self.failUnlessIsBarDotTxt)
3180         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3181         d.addCallback(self.failUnlessIsBarJSON)
3182         return d
3183
3184     def test_POST_rename_file_no_replace_queryarg(self):
3185         # rename a file and replace a directory with it
3186         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3187                       from_name="bar.txt", to_name='empty')
3188         d.addBoth(self.shouldFail, error.Error,
3189                   "POST_rename_file_no_replace_queryarg",
3190                   "409 Conflict",
3191                   "There was already a child by that name, and you asked me "
3192                   "to not replace it")
3193         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3194         d.addCallback(self.failUnlessIsEmptyJSON)
3195         return d
3196
3197     def test_POST_rename_file_no_replace_field(self):
3198         # rename a file and replace a directory with it
3199         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3200                       from_name="bar.txt", to_name='empty')
3201         d.addBoth(self.shouldFail, error.Error,
3202                   "POST_rename_file_no_replace_field",
3203                   "409 Conflict",
3204                   "There was already a child by that name, and you asked me "
3205                   "to not replace it")
3206         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3207         d.addCallback(self.failUnlessIsEmptyJSON)
3208         return d
3209
3210     def failUnlessIsEmptyJSON(self, res):
3211         data = simplejson.loads(res)
3212         self.failUnlessEqual(data[0], "dirnode", data)
3213         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3214
3215     def test_POST_rename_file_slash_fail(self):
3216         d = self.POST(self.public_url + "/foo", t="rename",
3217                       from_name="bar.txt", to_name='kirk/spock.txt')
3218         d.addBoth(self.shouldFail, error.Error,
3219                   "test_POST_rename_file_slash_fail",
3220                   "400 Bad Request",
3221                   "to_name= may not contain a slash",
3222                   )
3223         d.addCallback(lambda res:
3224                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3225         return d
3226
3227     def test_POST_rename_dir(self):
3228         d = self.POST(self.public_url, t="rename",
3229                       from_name="foo", to_name='plunk')
3230         d.addCallback(lambda res:
3231                       self.failIfNodeHasChild(self.public_root, u"foo"))
3232         d.addCallback(lambda res:
3233                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3234         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3235         d.addCallback(self.failUnlessIsFooJSON)
3236         return d
3237
3238     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3239         """ If target is not None then the redirection has to go to target.  If
3240         statuscode is not None then the redirection has to be accomplished with
3241         that HTTP status code."""
3242         if not isinstance(res, failure.Failure):
3243             to_where = (target is None) and "somewhere" or ("to " + target)
3244             self.fail("%s: we were expecting to get redirected %s, not get an"
3245                       " actual page: %s" % (which, to_where, res))
3246         res.trap(error.PageRedirect)
3247         if statuscode is not None:
3248             self.failUnlessReallyEqual(res.value.status, statuscode,
3249                                        "%s: not a redirect" % which)
3250         if target is not None:
3251             # the PageRedirect does not seem to capture the uri= query arg
3252             # properly, so we can't check for it.
3253             realtarget = self.webish_url + target
3254             self.failUnlessReallyEqual(res.value.location, realtarget,
3255                                        "%s: wrong target" % which)
3256         return res.value.location
3257
3258     def test_GET_URI_form(self):
3259         base = "/uri?uri=%s" % self._bar_txt_uri
3260         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3261         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3262         d = self.GET(base)
3263         d.addBoth(self.shouldRedirect, targetbase)
3264         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3265         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3266         d.addCallback(lambda res: self.GET(base+"&t=json"))
3267         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3268         d.addCallback(self.log, "about to get file by uri")
3269         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3270         d.addCallback(self.failUnlessIsBarDotTxt)
3271         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3272         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3273                                            followRedirect=True))
3274         d.addCallback(self.failUnlessIsFooJSON)
3275         d.addCallback(self.log, "got dir by uri")
3276
3277         return d
3278
3279     def test_GET_URI_form_bad(self):
3280         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3281                              "400 Bad Request", "GET /uri requires uri=",
3282                              self.GET, "/uri")
3283         return d
3284
3285     def test_GET_rename_form(self):
3286         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3287                      followRedirect=True)
3288         def _check(res):
3289             self.failUnlessIn('name="when_done" value="."', res)
3290             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3291             self.failUnlessIn(FAVICON_MARKUP, res)
3292         d.addCallback(_check)
3293         return d
3294
3295     def log(self, res, msg):
3296         #print "MSG: %s  RES: %s" % (msg, res)
3297         log.msg(msg)
3298         return res
3299
3300     def test_GET_URI_URL(self):
3301         base = "/uri/%s" % self._bar_txt_uri
3302         d = self.GET(base)
3303         d.addCallback(self.failUnlessIsBarDotTxt)
3304         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3305         d.addCallback(self.failUnlessIsBarDotTxt)
3306         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3307         d.addCallback(self.failUnlessIsBarDotTxt)
3308         return d
3309
3310     def test_GET_URI_URL_dir(self):
3311         base = "/uri/%s?t=json" % self._foo_uri
3312         d = self.GET(base)
3313         d.addCallback(self.failUnlessIsFooJSON)
3314         return d
3315
3316     def test_GET_URI_URL_missing(self):
3317         base = "/uri/%s" % self._bad_file_uri
3318         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3319                                  http.GONE, None, "NotEnoughSharesError",
3320                                  self.GET, base)
3321         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3322         # here? we must arrange for a download to fail after target.open()
3323         # has been called, and then inspect the response to see that it is
3324         # shorter than we expected.
3325         return d
3326
3327     def test_PUT_DIRURL_uri(self):
3328         d = self.s.create_dirnode()
3329         def _made_dir(dn):
3330             new_uri = dn.get_uri()
3331             # replace /foo with a new (empty) directory
3332             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3333             d.addCallback(lambda res:
3334                           self.failUnlessReallyEqual(res.strip(), new_uri))
3335             d.addCallback(lambda res:
3336                           self.failUnlessRWChildURIIs(self.public_root,
3337                                                       u"foo",
3338                                                       new_uri))
3339             return d
3340         d.addCallback(_made_dir)
3341         return d
3342
3343     def test_PUT_DIRURL_uri_noreplace(self):
3344         d = self.s.create_dirnode()
3345         def _made_dir(dn):
3346             new_uri = dn.get_uri()
3347             # replace /foo with a new (empty) directory, but ask that
3348             # replace=false, so it should fail
3349             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3350                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3351                                  self.PUT,
3352                                  self.public_url + "/foo?t=uri&replace=false",
3353                                  new_uri)
3354             d.addCallback(lambda res:
3355                           self.failUnlessRWChildURIIs(self.public_root,
3356                                                       u"foo",
3357                                                       self._foo_uri))
3358             return d
3359         d.addCallback(_made_dir)
3360         return d
3361
3362     def test_PUT_DIRURL_bad_t(self):
3363         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3364                              "400 Bad Request", "PUT to a directory",
3365                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3366         d.addCallback(lambda res:
3367                       self.failUnlessRWChildURIIs(self.public_root,
3368                                                   u"foo",
3369                                                   self._foo_uri))
3370         return d
3371
3372     def test_PUT_NEWFILEURL_uri(self):
3373         contents, n, new_uri = self.makefile(8)
3374         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3375         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3376         d.addCallback(lambda res:
3377                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3378                                                       contents))
3379         return d
3380
3381     def test_PUT_NEWFILEURL_mdmf(self):
3382         new_contents = self.NEWFILE_CONTENTS * 300000
3383         d = self.PUT(self.public_url + \
3384                      "/foo/mdmf.txt?format=mdmf",
3385                      new_contents)
3386         d.addCallback(lambda ignored:
3387             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3388         def _got_json(json):
3389             data = simplejson.loads(json)
3390             data = data[1]
3391             self.failUnlessIn("format", data)
3392             self.failUnlessEqual(data["format"], "MDMF")
3393             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3394             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3395         d.addCallback(_got_json)
3396         return d
3397
3398     def test_PUT_NEWFILEURL_sdmf(self):
3399         new_contents = self.NEWFILE_CONTENTS * 300000
3400         d = self.PUT(self.public_url + \
3401                      "/foo/sdmf.txt?format=sdmf",
3402                      new_contents)
3403         d.addCallback(lambda ignored:
3404             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3405         def _got_json(json):
3406             data = simplejson.loads(json)
3407             data = data[1]
3408             self.failUnlessIn("format", data)
3409             self.failUnlessEqual(data["format"], "SDMF")
3410         d.addCallback(_got_json)
3411         return d
3412
3413     def test_PUT_NEWFILEURL_bad_format(self):
3414        new_contents = self.NEWFILE_CONTENTS * 300000
3415        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3416                                    400, "Bad Request", "Unknown format: foo",
3417                                    self.PUT, self.public_url + \
3418                                    "/foo/foo.txt?format=foo",
3419                                    new_contents)
3420
3421     def test_PUT_NEWFILEURL_uri_replace(self):
3422         contents, n, new_uri = self.makefile(8)
3423         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3424         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3425         d.addCallback(lambda res:
3426                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3427                                                       contents))
3428         return d
3429
3430     def test_PUT_NEWFILEURL_uri_no_replace(self):
3431         contents, n, new_uri = self.makefile(8)
3432         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3433         d.addBoth(self.shouldFail, error.Error,
3434                   "PUT_NEWFILEURL_uri_no_replace",
3435                   "409 Conflict",
3436                   "There was already a child by that name, and you asked me "
3437                   "to not replace it")
3438         return d
3439
3440     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3441         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3442         d.addBoth(self.shouldFail, error.Error,
3443                   "POST_put_uri_unknown_bad",
3444                   "400 Bad Request",
3445                   "unknown cap in a write slot")
3446         return d
3447
3448     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3449         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3450         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3451                       u"put-future-ro.txt")
3452         return d
3453
3454     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3455         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3456         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3457                       u"put-future-imm.txt")
3458         return d
3459
3460     def test_PUT_NEWFILE_URI(self):
3461         file_contents = "New file contents here\n"
3462         d = self.PUT("/uri", file_contents)
3463         def _check(uri):
3464             assert isinstance(uri, str), uri
3465             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3466             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3467                                        file_contents)
3468             return self.GET("/uri/%s" % uri)
3469         d.addCallback(_check)
3470         def _check2(res):
3471             self.failUnlessReallyEqual(res, file_contents)
3472         d.addCallback(_check2)
3473         return d
3474
3475     def test_PUT_NEWFILE_URI_not_mutable(self):
3476         file_contents = "New file contents here\n"
3477         d = self.PUT("/uri?mutable=false", file_contents)
3478         def _check(uri):
3479             assert isinstance(uri, str), uri
3480             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3481             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3482                                        file_contents)
3483             return self.GET("/uri/%s" % uri)
3484         d.addCallback(_check)
3485         def _check2(res):
3486             self.failUnlessReallyEqual(res, file_contents)
3487         d.addCallback(_check2)
3488         return d
3489
3490     def test_PUT_NEWFILE_URI_only_PUT(self):
3491         d = self.PUT("/uri?t=bogus", "")
3492         d.addBoth(self.shouldFail, error.Error,
3493                   "PUT_NEWFILE_URI_only_PUT",
3494                   "400 Bad Request",
3495                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3496         return d
3497
3498     def test_PUT_NEWFILE_URI_mutable(self):
3499         file_contents = "New file contents here\n"
3500         d = self.PUT("/uri?mutable=true", file_contents)
3501         def _check1(filecap):
3502             filecap = filecap.strip()
3503             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3504             self.filecap = filecap
3505             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3506             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3507             n = self.s.create_node_from_uri(filecap)
3508             return n.download_best_version()
3509         d.addCallback(_check1)
3510         def _check2(data):
3511             self.failUnlessReallyEqual(data, file_contents)
3512             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3513         d.addCallback(_check2)
3514         def _check3(res):
3515             self.failUnlessReallyEqual(res, file_contents)
3516         d.addCallback(_check3)
3517         return d
3518
3519     def test_PUT_mkdir(self):
3520         d = self.PUT("/uri?t=mkdir", "")
3521         def _check(uri):
3522             n = self.s.create_node_from_uri(uri.strip())
3523             d2 = self.failUnlessNodeKeysAre(n, [])
3524             d2.addCallback(lambda res:
3525                            self.GET("/uri/%s?t=json" % uri))
3526             return d2
3527         d.addCallback(_check)
3528         d.addCallback(self.failUnlessIsEmptyJSON)
3529         return d
3530
3531     def test_PUT_mkdir_mdmf(self):
3532         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3533         def _got(res):
3534             u = uri.from_string(res)
3535             # Check that this is an MDMF writecap
3536             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3537         d.addCallback(_got)
3538         return d
3539
3540     def test_PUT_mkdir_sdmf(self):
3541         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3542         def _got(res):
3543             u = uri.from_string(res)
3544             self.failUnlessIsInstance(u, uri.DirectoryURI)
3545         d.addCallback(_got)
3546         return d
3547
3548     def test_PUT_mkdir_bad_format(self):
3549         return self.shouldHTTPError("PUT_mkdir_bad_format",
3550                                     400, "Bad Request", "Unknown format: foo",
3551                                     self.PUT, "/uri?t=mkdir&format=foo",
3552                                     "")
3553
3554     def test_POST_check(self):
3555         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3556         def _done(res):
3557             # this returns a string form of the results, which are probably
3558             # None since we're using fake filenodes.
3559             # TODO: verify that the check actually happened, by changing
3560             # FakeCHKFileNode to count how many times .check() has been
3561             # called.
3562             pass
3563         d.addCallback(_done)
3564         return d
3565
3566
3567     def test_PUT_update_at_offset(self):
3568         file_contents = "test file" * 100000 # about 900 KiB
3569         d = self.PUT("/uri?mutable=true", file_contents)
3570         def _then(filecap):
3571             self.filecap = filecap
3572             new_data = file_contents[:100]
3573             new = "replaced and so on"
3574             new_data += new
3575             new_data += file_contents[len(new_data):]
3576             assert len(new_data) == len(file_contents)
3577             self.new_data = new_data
3578         d.addCallback(_then)
3579         d.addCallback(lambda ignored:
3580             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3581                      "replaced and so on"))
3582         def _get_data(filecap):
3583             n = self.s.create_node_from_uri(filecap)
3584             return n.download_best_version()
3585         d.addCallback(_get_data)
3586         d.addCallback(lambda results:
3587             self.failUnlessEqual(results, self.new_data))
3588         # Now try appending things to the file
3589         d.addCallback(lambda ignored:
3590             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3591                      "puppies" * 100))
3592         d.addCallback(_get_data)
3593         d.addCallback(lambda results:
3594             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3595         # and try replacing the beginning of the file
3596         d.addCallback(lambda ignored:
3597             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3598         d.addCallback(_get_data)
3599         d.addCallback(lambda results:
3600             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3601         return d
3602
3603     def test_PUT_update_at_invalid_offset(self):
3604         file_contents = "test file" * 100000 # about 900 KiB
3605         d = self.PUT("/uri?mutable=true", file_contents)
3606         def _then(filecap):
3607             self.filecap = filecap
3608         d.addCallback(_then)
3609         # Negative offsets should cause an error.
3610         d.addCallback(lambda ignored:
3611             self.shouldHTTPError("PUT_update_at_invalid_offset",
3612                                  400, "Bad Request",
3613                                  "Invalid offset",
3614                                  self.PUT,
3615                                  "/uri/%s?offset=-1" % self.filecap,
3616                                  "foo"))
3617         return d
3618
3619     def test_PUT_update_at_offset_immutable(self):
3620         file_contents = "Test file" * 100000
3621         d = self.PUT("/uri", file_contents)
3622         def _then(filecap):
3623             self.filecap = filecap
3624         d.addCallback(_then)
3625         d.addCallback(lambda ignored:
3626             self.shouldHTTPError("PUT_update_at_offset_immutable",
3627                                  400, "Bad Request",
3628                                  "immutable",
3629                                  self.PUT,
3630                                  "/uri/%s?offset=50" % self.filecap,
3631                                  "foo"))
3632         return d
3633
3634
3635     def test_bad_method(self):
3636         url = self.webish_url + self.public_url + "/foo/bar.txt"
3637         d = self.shouldHTTPError("bad_method",
3638                                  501, "Not Implemented",
3639                                  "I don't know how to treat a BOGUS request.",
3640                                  client.getPage, url, method="BOGUS")
3641         return d
3642
3643     def test_short_url(self):
3644         url = self.webish_url + "/uri"
3645         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3646                                  "I don't know how to treat a DELETE request.",
3647                                  client.getPage, url, method="DELETE")
3648         return d
3649
3650     def test_ophandle_bad(self):
3651         url = self.webish_url + "/operations/bogus?t=status"
3652         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3653                                  "unknown/expired handle 'bogus'",
3654                                  client.getPage, url)
3655         return d
3656
3657     def test_ophandle_cancel(self):
3658         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3659                       followRedirect=True)
3660         d.addCallback(lambda ignored:
3661                       self.GET("/operations/128?t=status&output=JSON"))
3662         def _check1(res):
3663             data = simplejson.loads(res)
3664             self.failUnless("finished" in data, res)
3665             monitor = self.ws.root.child_operations.handles["128"][0]
3666             d = self.POST("/operations/128?t=cancel&output=JSON")
3667             def _check2(res):
3668                 data = simplejson.loads(res)
3669                 self.failUnless("finished" in data, res)
3670                 # t=cancel causes the handle to be forgotten
3671                 self.failUnless(monitor.is_cancelled())
3672             d.addCallback(_check2)
3673             return d
3674         d.addCallback(_check1)
3675         d.addCallback(lambda ignored:
3676                       self.shouldHTTPError("ophandle_cancel",
3677                                            404, "404 Not Found",
3678                                            "unknown/expired handle '128'",
3679                                            self.GET,
3680                                            "/operations/128?t=status&output=JSON"))
3681         return d
3682
3683     def test_ophandle_retainfor(self):
3684         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3685                       followRedirect=True)
3686         d.addCallback(lambda ignored:
3687                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3688         def _check1(res):
3689             data = simplejson.loads(res)
3690             self.failUnless("finished" in data, res)
3691         d.addCallback(_check1)
3692         # the retain-for=0 will cause the handle to be expired very soon
3693         d.addCallback(lambda ign:
3694             self.clock.advance(2.0))
3695         d.addCallback(lambda ignored:
3696                       self.shouldHTTPError("ophandle_retainfor",
3697                                            404, "404 Not Found",
3698                                            "unknown/expired handle '129'",
3699                                            self.GET,
3700                                            "/operations/129?t=status&output=JSON"))
3701         return d
3702
3703     def test_ophandle_release_after_complete(self):
3704         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3705                       followRedirect=True)
3706         d.addCallback(self.wait_for_operation, "130")
3707         d.addCallback(lambda ignored:
3708                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3709         # the release-after-complete=true will cause the handle to be expired
3710         d.addCallback(lambda ignored:
3711                       self.shouldHTTPError("ophandle_release_after_complete",
3712                                            404, "404 Not Found",
3713                                            "unknown/expired handle '130'",
3714                                            self.GET,
3715                                            "/operations/130?t=status&output=JSON"))
3716         return d
3717
3718     def test_uncollected_ophandle_expiration(self):
3719         # uncollected ophandles should expire after 4 days
3720         def _make_uncollected_ophandle(ophandle):
3721             d = self.POST(self.public_url +
3722                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3723                           followRedirect=False)
3724             # When we start the operation, the webapi server will want
3725             # to redirect us to the page for the ophandle, so we get
3726             # confirmation that the operation has started. If the
3727             # manifest operation has finished by the time we get there,
3728             # following that redirect (by setting followRedirect=True
3729             # above) has the side effect of collecting the ophandle that
3730             # we've just created, which means that we can't use the
3731             # ophandle to test the uncollected timeout anymore. So,
3732             # instead, catch the 302 here and don't follow it.
3733             d.addBoth(self.should302, "uncollected_ophandle_creation")
3734             return d
3735         # Create an ophandle, don't collect it, then advance the clock by
3736         # 4 days - 1 second and make sure that the ophandle is still there.
3737         d = _make_uncollected_ophandle(131)
3738         d.addCallback(lambda ign:
3739             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3740         d.addCallback(lambda ign:
3741             self.GET("/operations/131?t=status&output=JSON"))
3742         def _check1(res):
3743             data = simplejson.loads(res)
3744             self.failUnless("finished" in data, res)
3745         d.addCallback(_check1)
3746         # Create an ophandle, don't collect it, then try to collect it
3747         # after 4 days. It should be gone.
3748         d.addCallback(lambda ign:
3749             _make_uncollected_ophandle(132))
3750         d.addCallback(lambda ign:
3751             self.clock.advance(96*60*60))
3752         d.addCallback(lambda ign:
3753             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3754                                  404, "404 Not Found",
3755                                  "unknown/expired handle '132'",
3756                                  self.GET,
3757                                  "/operations/132?t=status&output=JSON"))
3758         return d
3759
3760     def test_collected_ophandle_expiration(self):
3761         # collected ophandles should expire after 1 day
3762         def _make_collected_ophandle(ophandle):
3763             d = self.POST(self.public_url +
3764                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3765                           followRedirect=True)
3766             # By following the initial redirect, we collect the ophandle
3767             # we've just created.
3768             return d
3769         # Create a collected ophandle, then collect it after 23 hours
3770         # and 59 seconds to make sure that it is still there.
3771         d = _make_collected_ophandle(133)
3772         d.addCallback(lambda ign:
3773             self.clock.advance((24*60*60) - 1))
3774         d.addCallback(lambda ign:
3775             self.GET("/operations/133?t=status&output=JSON"))
3776         def _check1(res):
3777             data = simplejson.loads(res)
3778             self.failUnless("finished" in data, res)
3779         d.addCallback(_check1)
3780         # Create another uncollected ophandle, then try to collect it
3781         # after 24 hours to make sure that it is gone.
3782         d.addCallback(lambda ign:
3783             _make_collected_ophandle(134))
3784         d.addCallback(lambda ign:
3785             self.clock.advance(24*60*60))
3786         d.addCallback(lambda ign:
3787             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3788                                  404, "404 Not Found",
3789                                  "unknown/expired handle '134'",
3790                                  self.GET,
3791                                  "/operations/134?t=status&output=JSON"))
3792         return d
3793
3794     def test_incident(self):
3795         d = self.POST("/report_incident", details="eek")
3796         def _done(res):
3797             self.failIfIn("<html>", res)
3798             self.failUnlessIn("Thank you for your report!", res)
3799         d.addCallback(_done)
3800         return d
3801
3802     def test_static(self):
3803         webdir = os.path.join(self.staticdir, "subdir")
3804         fileutil.make_dirs(webdir)
3805         f = open(os.path.join(webdir, "hello.txt"), "wb")
3806         f.write("hello")
3807         f.close()
3808
3809         d = self.GET("/static/subdir/hello.txt")
3810         def _check(res):
3811             self.failUnlessReallyEqual(res, "hello")
3812         d.addCallback(_check)
3813         return d
3814
3815
3816 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3817     def test_load_file(self):
3818         # This will raise an exception unless a well-formed XML file is found under that name.
3819         common.getxmlfile('directory.xhtml').load()
3820
3821     def test_parse_replace_arg(self):
3822         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3823         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3824         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3825                                    "only-files")
3826         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3827                         common.parse_replace_arg, "only_fles")
3828
3829     def test_abbreviate_time(self):
3830         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3831         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3832         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3833         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3834         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3835         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3836
3837     def test_compute_rate(self):
3838         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3839         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3840         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3841         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3842         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3843         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3844         self.shouldFail(AssertionError, "test_compute_rate", "",
3845                         common.compute_rate, -100, 10)
3846         self.shouldFail(AssertionError, "test_compute_rate", "",
3847                         common.compute_rate, 100, -10)
3848
3849         # Sanity check
3850         rate = common.compute_rate(10*1000*1000, 1)
3851         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3852
3853     def test_abbreviate_rate(self):
3854         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3855         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3856         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3857         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3858
3859     def test_abbreviate_size(self):
3860         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3861         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3862         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3863         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3864         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3865
3866     def test_plural(self):
3867         def convert(s):
3868             return "%d second%s" % (s, status.plural(s))
3869         self.failUnlessReallyEqual(convert(0), "0 seconds")
3870         self.failUnlessReallyEqual(convert(1), "1 second")
3871         self.failUnlessReallyEqual(convert(2), "2 seconds")
3872         def convert2(s):
3873             return "has share%s: %s" % (status.plural(s), ",".join(s))
3874         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3875         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3876         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3877
3878
3879 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3880
3881     def CHECK(self, ign, which, args, clientnum=0):
3882         fileurl = self.fileurls[which]
3883         url = fileurl + "?" + args
3884         return self.GET(url, method="POST", clientnum=clientnum)
3885
3886     def test_filecheck(self):
3887         self.basedir = "web/Grid/filecheck"
3888         self.set_up_grid()
3889         c0 = self.g.clients[0]
3890         self.uris = {}
3891         DATA = "data" * 100
3892         d = c0.upload(upload.Data(DATA, convergence=""))
3893         def _stash_uri(ur, which):
3894             self.uris[which] = ur.uri
3895         d.addCallback(_stash_uri, "good")
3896         d.addCallback(lambda ign:
3897                       c0.upload(upload.Data(DATA+"1", convergence="")))
3898         d.addCallback(_stash_uri, "sick")
3899         d.addCallback(lambda ign:
3900                       c0.upload(upload.Data(DATA+"2", convergence="")))
3901         d.addCallback(_stash_uri, "dead")
3902         def _stash_mutable_uri(n, which):
3903             self.uris[which] = n.get_uri()
3904             assert isinstance(self.uris[which], str)
3905         d.addCallback(lambda ign:
3906             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3907         d.addCallback(_stash_mutable_uri, "corrupt")
3908         d.addCallback(lambda ign:
3909                       c0.upload(upload.Data("literal", convergence="")))
3910         d.addCallback(_stash_uri, "small")
3911         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3912         d.addCallback(_stash_mutable_uri, "smalldir")
3913
3914         def _compute_fileurls(ignored):
3915             self.fileurls = {}
3916             for which in self.uris:
3917                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3918         d.addCallback(_compute_fileurls)
3919
3920         def _clobber_shares(ignored):
3921             good_shares = self.find_uri_shares(self.uris["good"])
3922             self.failUnlessReallyEqual(len(good_shares), 10)
3923             sick_shares = self.find_uri_shares(self.uris["sick"])
3924             os.unlink(sick_shares[0][2])
3925             dead_shares = self.find_uri_shares(self.uris["dead"])
3926             for i in range(1, 10):
3927                 os.unlink(dead_shares[i][2])
3928             c_shares = self.find_uri_shares(self.uris["corrupt"])
3929             cso = CorruptShareOptions()
3930             cso.stdout = StringIO()
3931             cso.parseOptions([c_shares[0][2]])
3932             corrupt_share(cso)
3933         d.addCallback(_clobber_shares)
3934
3935         d.addCallback(self.CHECK, "good", "t=check")
3936         def _got_html_good(res):
3937             self.failUnlessIn("Healthy", res)
3938             self.failIfIn("Not Healthy", res)
3939             self.failUnlessIn(FAVICON_MARKUP, res)
3940         d.addCallback(_got_html_good)
3941         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3942         def _got_html_good_return_to(res):
3943             self.failUnlessIn("Healthy", res)
3944             self.failIfIn("Not Healthy", res)
3945             self.failUnlessIn('<a href="somewhere">Return to file', res)
3946         d.addCallback(_got_html_good_return_to)
3947         d.addCallback(self.CHECK, "good", "t=check&output=json")
3948         def _got_json_good(res):
3949             r = simplejson.loads(res)
3950             self.failUnlessEqual(r["summary"], "Healthy")
3951             self.failUnless(r["results"]["healthy"])
3952             self.failIf(r["results"]["needs-rebalancing"])
3953             self.failUnless(r["results"]["recoverable"])
3954         d.addCallback(_got_json_good)
3955
3956         d.addCallback(self.CHECK, "small", "t=check")
3957         def _got_html_small(res):
3958             self.failUnlessIn("Literal files are always healthy", res)
3959             self.failIfIn("Not Healthy", res)
3960         d.addCallback(_got_html_small)
3961         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3962         def _got_html_small_return_to(res):
3963             self.failUnlessIn("Literal files are always healthy", res)
3964             self.failIfIn("Not Healthy", res)
3965             self.failUnlessIn('<a href="somewhere">Return to file', res)
3966         d.addCallback(_got_html_small_return_to)
3967         d.addCallback(self.CHECK, "small", "t=check&output=json")
3968         def _got_json_small(res):
3969             r = simplejson.loads(res)
3970             self.failUnlessEqual(r["storage-index"], "")
3971             self.failUnless(r["results"]["healthy"])
3972         d.addCallback(_got_json_small)
3973
3974         d.addCallback(self.CHECK, "smalldir", "t=check")
3975         def _got_html_smalldir(res):
3976             self.failUnlessIn("Literal files are always healthy", res)
3977             self.failIfIn("Not Healthy", res)
3978         d.addCallback(_got_html_smalldir)
3979         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3980         def _got_json_smalldir(res):
3981             r = simplejson.loads(res)
3982             self.failUnlessEqual(r["storage-index"], "")
3983             self.failUnless(r["results"]["healthy"])
3984         d.addCallback(_got_json_smalldir)
3985
3986         d.addCallback(self.CHECK, "sick", "t=check")
3987         def _got_html_sick(res):
3988             self.failUnlessIn("Not Healthy", res)
3989         d.addCallback(_got_html_sick)
3990         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3991         def _got_json_sick(res):
3992             r = simplejson.loads(res)
3993             self.failUnlessEqual(r["summary"],
3994                                  "Not Healthy: 9 shares (enc 3-of-10)")
3995             self.failIf(r["results"]["healthy"])
3996             self.failIf(r["results"]["needs-rebalancing"])
3997             self.failUnless(r["results"]["recoverable"])
3998         d.addCallback(_got_json_sick)
3999
4000         d.addCallback(self.CHECK, "dead", "t=check")
4001         def _got_html_dead(res):
4002             self.failUnlessIn("Not Healthy", res)
4003         d.addCallback(_got_html_dead)
4004         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4005         def _got_json_dead(res):
4006             r = simplejson.loads(res)
4007             self.failUnlessEqual(r["summary"],
4008                                  "Not Healthy: 1 shares (enc 3-of-10)")
4009             self.failIf(r["results"]["healthy"])
4010             self.failIf(r["results"]["needs-rebalancing"])
4011             self.failIf(r["results"]["recoverable"])
4012         d.addCallback(_got_json_dead)
4013
4014         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4015         def _got_html_corrupt(res):
4016             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4017         d.addCallback(_got_html_corrupt)
4018         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4019         def _got_json_corrupt(res):
4020             r = simplejson.loads(res)
4021             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4022             self.failIf(r["results"]["healthy"])
4023             self.failUnless(r["results"]["recoverable"])
4024             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4025             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4026         d.addCallback(_got_json_corrupt)
4027
4028         d.addErrback(self.explain_web_error)
4029         return d
4030
4031     def test_repair_html(self):
4032         self.basedir = "web/Grid/repair_html"
4033         self.set_up_grid()
4034         c0 = self.g.clients[0]
4035         self.uris = {}
4036         DATA = "data" * 100
4037         d = c0.upload(upload.Data(DATA, convergence=""))
4038         def _stash_uri(ur, which):
4039             self.uris[which] = ur.uri
4040         d.addCallback(_stash_uri, "good")
4041         d.addCallback(lambda ign:
4042                       c0.upload(upload.Data(DATA+"1", convergence="")))
4043         d.addCallback(_stash_uri, "sick")
4044         d.addCallback(lambda ign:
4045                       c0.upload(upload.Data(DATA+"2", convergence="")))
4046         d.addCallback(_stash_uri, "dead")
4047         def _stash_mutable_uri(n, which):
4048             self.uris[which] = n.get_uri()
4049             assert isinstance(self.uris[which], str)
4050         d.addCallback(lambda ign:
4051             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4052         d.addCallback(_stash_mutable_uri, "corrupt")
4053
4054         def _compute_fileurls(ignored):
4055             self.fileurls = {}
4056             for which in self.uris:
4057                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4058         d.addCallback(_compute_fileurls)
4059
4060         def _clobber_shares(ignored):
4061             good_shares = self.find_uri_shares(self.uris["good"])
4062             self.failUnlessReallyEqual(len(good_shares), 10)
4063             sick_shares = self.find_uri_shares(self.uris["sick"])
4064             os.unlink(sick_shares[0][2])
4065             dead_shares = self.find_uri_shares(self.uris["dead"])
4066             for i in range(1, 10):
4067                 os.unlink(dead_shares[i][2])
4068             c_shares = self.find_uri_shares(self.uris["corrupt"])
4069             cso = CorruptShareOptions()
4070             cso.stdout = StringIO()
4071             cso.parseOptions([c_shares[0][2]])
4072             corrupt_share(cso)
4073         d.addCallback(_clobber_shares)
4074
4075         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4076         def _got_html_good(res):
4077             self.failUnlessIn("Healthy", res)
4078             self.failIfIn("Not Healthy", res)
4079             self.failUnlessIn("No repair necessary", res)
4080             self.failUnlessIn(FAVICON_MARKUP, res)
4081         d.addCallback(_got_html_good)
4082
4083         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4084         def _got_html_sick(res):
4085             self.failUnlessIn("Healthy : healthy", res)
4086             self.failIfIn("Not Healthy", res)
4087             self.failUnlessIn("Repair successful", res)
4088         d.addCallback(_got_html_sick)
4089
4090         # repair of a dead file will fail, of course, but it isn't yet
4091         # clear how this should be reported. Right now it shows up as
4092         # a "410 Gone".
4093         #
4094         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4095         #def _got_html_dead(res):
4096         #    print res
4097         #    self.failUnlessIn("Healthy : healthy", res)
4098         #    self.failIfIn("Not Healthy", res)
4099         #    self.failUnlessIn("No repair necessary", res)
4100         #d.addCallback(_got_html_dead)
4101
4102         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4103         def _got_html_corrupt(res):
4104             self.failUnlessIn("Healthy : Healthy", res)
4105             self.failIfIn("Not Healthy", res)
4106             self.failUnlessIn("Repair successful", res)
4107         d.addCallback(_got_html_corrupt)
4108
4109         d.addErrback(self.explain_web_error)
4110         return d
4111
4112     def test_repair_json(self):
4113         self.basedir = "web/Grid/repair_json"
4114         self.set_up_grid()
4115         c0 = self.g.clients[0]
4116         self.uris = {}
4117         DATA = "data" * 100
4118         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4119         def _stash_uri(ur, which):
4120             self.uris[which] = ur.uri
4121         d.addCallback(_stash_uri, "sick")
4122
4123         def _compute_fileurls(ignored):
4124             self.fileurls = {}
4125             for which in self.uris:
4126                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4127         d.addCallback(_compute_fileurls)
4128
4129         def _clobber_shares(ignored):
4130             sick_shares = self.find_uri_shares(self.uris["sick"])
4131             os.unlink(sick_shares[0][2])
4132         d.addCallback(_clobber_shares)
4133
4134         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4135         def _got_json_sick(res):
4136             r = simplejson.loads(res)
4137             self.failUnlessReallyEqual(r["repair-attempted"], True)
4138             self.failUnlessReallyEqual(r["repair-successful"], True)
4139             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4140                                  "Not Healthy: 9 shares (enc 3-of-10)")
4141             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4142             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4143             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4144         d.addCallback(_got_json_sick)
4145
4146         d.addErrback(self.explain_web_error)
4147         return d
4148
4149     def test_unknown(self, immutable=False):
4150         self.basedir = "web/Grid/unknown"
4151         if immutable:
4152             self.basedir = "web/Grid/unknown-immutable"
4153
4154         self.set_up_grid()
4155         c0 = self.g.clients[0]
4156         self.uris = {}
4157         self.fileurls = {}
4158
4159         # the future cap format may contain slashes, which must be tolerated
4160         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4161                                                            safe="")
4162
4163         if immutable:
4164             name = u"future-imm"
4165             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4166             d = c0.create_immutable_dirnode({name: (future_node, {})})
4167         else:
4168             name = u"future"
4169             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4170             d = c0.create_dirnode()
4171
4172         def _stash_root_and_create_file(n):
4173             self.rootnode = n
4174             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4175             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4176             if not immutable:
4177                 return self.rootnode.set_node(name, future_node)
4178         d.addCallback(_stash_root_and_create_file)
4179
4180         # make sure directory listing tolerates unknown nodes
4181         d.addCallback(lambda ign: self.GET(self.rooturl))
4182         def _check_directory_html(res, expected_type_suffix):
4183             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4184                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4185                                  re.DOTALL)
4186             self.failUnless(re.search(pattern, res), res)
4187             # find the More Info link for name, should be relative
4188             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4189             info_url = mo.group(1)
4190             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4191         if immutable:
4192             d.addCallback(_check_directory_html, "-IMM")
4193         else:
4194             d.addCallback(_check_directory_html, "")
4195
4196         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4197         def _check_directory_json(res, expect_rw_uri):
4198             data = simplejson.loads(res)
4199             self.failUnlessEqual(data[0], "dirnode")
4200             f = data[1]["children"][name]
4201             self.failUnlessEqual(f[0], "unknown")
4202             if expect_rw_uri:
4203                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4204             else:
4205                 self.failIfIn("rw_uri", f[1])
4206             if immutable:
4207                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4208             else:
4209                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4210             self.failUnlessIn("metadata", f[1])
4211         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4212
4213         def _check_info(res, expect_rw_uri, expect_ro_uri):
4214             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4215             if expect_rw_uri:
4216                 self.failUnlessIn(unknown_rwcap, res)
4217             if expect_ro_uri:
4218                 if immutable:
4219                     self.failUnlessIn(unknown_immcap, res)
4220                 else:
4221                     self.failUnlessIn(unknown_rocap, res)
4222             else:
4223                 self.failIfIn(unknown_rocap, res)
4224             self.failIfIn("Raw data as", res)
4225             self.failIfIn("Directory writecap", res)
4226             self.failIfIn("Checker Operations", res)
4227             self.failIfIn("Mutable File Operations", res)
4228             self.failIfIn("Directory Operations", res)
4229
4230         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4231         # why they fail. Possibly related to ticket #922.
4232
4233         d.addCallback(lambda ign: self.GET(expected_info_url))
4234         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4235         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4236         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4237
4238         def _check_json(res, expect_rw_uri):
4239             data = simplejson.loads(res)
4240             self.failUnlessEqual(data[0], "unknown")
4241             if expect_rw_uri:
4242                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4243             else:
4244                 self.failIfIn("rw_uri", data[1])
4245
4246             if immutable:
4247                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4248                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4249             elif expect_rw_uri:
4250                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4251                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4252             else:
4253                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4254                 self.failIfIn("mutable", data[1])
4255
4256             # TODO: check metadata contents
4257             self.failUnlessIn("metadata", data[1])
4258
4259         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4260         d.addCallback(_check_json, expect_rw_uri=not immutable)
4261
4262         # and make sure that a read-only version of the directory can be
4263         # rendered too. This version will not have unknown_rwcap, whether
4264         # or not future_node was immutable.
4265         d.addCallback(lambda ign: self.GET(self.rourl))
4266         if immutable:
4267             d.addCallback(_check_directory_html, "-IMM")
4268         else:
4269             d.addCallback(_check_directory_html, "-RO")
4270
4271         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4272         d.addCallback(_check_directory_json, expect_rw_uri=False)
4273
4274         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4275         d.addCallback(_check_json, expect_rw_uri=False)
4276
4277         # TODO: check that getting t=info from the Info link in the ro directory
4278         # works, and does not include the writecap URI.
4279         return d
4280
4281     def test_immutable_unknown(self):
4282         return self.test_unknown(immutable=True)
4283
4284     def test_mutant_dirnodes_are_omitted(self):
4285         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4286
4287         self.set_up_grid()
4288         c = self.g.clients[0]
4289         nm = c.nodemaker
4290         self.uris = {}
4291         self.fileurls = {}
4292
4293         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4294         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4295         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4296
4297         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4298         # test the dirnode and web layers separately.
4299
4300         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4301         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4302         # When the directory is read, the mutants should be silently disposed of, leaving
4303         # their lonely sibling.
4304         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4305         # because immutable directories don't have a writecap and therefore that field
4306         # isn't (and can't be) decrypted.
4307         # TODO: The field still exists in the netstring. Technically we should check what
4308         # happens if something is put there (_unpack_contents should raise ValueError),
4309         # but that can wait.
4310
4311         lonely_child = nm.create_from_cap(lonely_uri)
4312         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4313         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4314
4315         def _by_hook_or_by_crook():
4316             return True
4317         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4318             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4319
4320         mutant_write_in_ro_child.get_write_uri    = lambda: None
4321         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4322
4323         kids = {u"lonely":      (lonely_child, {}),
4324                 u"ro":          (mutant_ro_child, {}),
4325                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4326                 }
4327         d = c.create_immutable_dirnode(kids)
4328
4329         def _created(dn):
4330             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4331             self.failIf(dn.is_mutable())
4332             self.failUnless(dn.is_readonly())
4333             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4334             self.failIf(hasattr(dn._node, 'get_writekey'))
4335             rep = str(dn)
4336             self.failUnlessIn("RO-IMM", rep)
4337             cap = dn.get_cap()
4338             self.failUnlessIn("CHK", cap.to_string())
4339             self.cap = cap
4340             self.rootnode = dn
4341             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4342             return download_to_data(dn._node)
4343         d.addCallback(_created)
4344
4345         def _check_data(data):
4346             # Decode the netstring representation of the directory to check that all children
4347             # are present. This is a bit of an abstraction violation, but there's not really
4348             # any other way to do it given that the real DirectoryNode._unpack_contents would
4349             # strip the mutant children out (which is what we're trying to test, later).
4350             position = 0
4351             numkids = 0
4352             while position < len(data):
4353                 entries, position = split_netstring(data, 1, position)
4354                 entry = entries[0]
4355                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4356                 name = name_utf8.decode("utf-8")
4357                 self.failUnlessEqual(rwcapdata, "")
4358                 self.failUnlessIn(name, kids)
4359                 (expected_child, ign) = kids[name]
4360                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4361                 numkids += 1
4362
4363             self.failUnlessReallyEqual(numkids, 3)
4364             return self.rootnode.list()
4365         d.addCallback(_check_data)
4366
4367         # Now when we use the real directory listing code, the mutants should be absent.
4368         def _check_kids(children):
4369             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4370             lonely_node, lonely_metadata = children[u"lonely"]
4371
4372             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4373             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4374         d.addCallback(_check_kids)
4375
4376         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4377         d.addCallback(lambda n: n.list())
4378         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4379
4380         # Make sure the lonely child can be listed in HTML...
4381         d.addCallback(lambda ign: self.GET(self.rooturl))
4382         def _check_html(res):
4383             self.failIfIn("URI:SSK", res)
4384             get_lonely = "".join([r'<td>FILE</td>',
4385                                   r'\s+<td>',
4386                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4387                                   r'</td>',
4388                                   r'\s+<td align="right">%d</td>' % len("one"),
4389                                   ])
4390             self.failUnless(re.search(get_lonely, res), res)
4391
4392             # find the More Info link for name, should be relative
4393             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4394             info_url = mo.group(1)
4395             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4396         d.addCallback(_check_html)
4397
4398         # ... and in JSON.
4399         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4400         def _check_json(res):
4401             data = simplejson.loads(res)
4402             self.failUnlessEqual(data[0], "dirnode")
4403             listed_children = data[1]["children"]
4404             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4405             ll_type, ll_data = listed_children[u"lonely"]
4406             self.failUnlessEqual(ll_type, "filenode")
4407             self.failIfIn("rw_uri", ll_data)
4408             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4409         d.addCallback(_check_json)
4410         return d
4411
4412     def test_deep_check(self):
4413         self.basedir = "web/Grid/deep_check"
4414         self.set_up_grid()
4415         c0 = self.g.clients[0]
4416         self.uris = {}
4417         self.fileurls = {}
4418         DATA = "data" * 100
4419         d = c0.create_dirnode()
4420         def _stash_root_and_create_file(n):
4421             self.rootnode = n
4422             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4423             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4424         d.addCallback(_stash_root_and_create_file)
4425         def _stash_uri(fn, which):
4426             self.uris[which] = fn.get_uri()
4427             return fn
4428         d.addCallback(_stash_uri, "good")
4429         d.addCallback(lambda ign:
4430                       self.rootnode.add_file(u"small",
4431                                              upload.Data("literal",
4432                                                         convergence="")))
4433         d.addCallback(_stash_uri, "small")
4434         d.addCallback(lambda ign:
4435                       self.rootnode.add_file(u"sick",
4436                                              upload.Data(DATA+"1",
4437                                                         convergence="")))
4438         d.addCallback(_stash_uri, "sick")
4439
4440         # this tests that deep-check and stream-manifest will ignore
4441         # UnknownNode instances. Hopefully this will also cover deep-stats.
4442         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4443         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4444
4445         def _clobber_shares(ignored):
4446             self.delete_shares_numbered(self.uris["sick"], [0,1])
4447         d.addCallback(_clobber_shares)
4448
4449         # root
4450         # root/good
4451         # root/small
4452         # root/sick
4453         # root/future
4454
4455         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4456         def _done(res):
4457             try:
4458                 units = [simplejson.loads(line)
4459                          for line in res.splitlines()
4460                          if line]
4461             except ValueError:
4462                 print "response is:", res
4463                 print "undecodeable line was '%s'" % line
4464                 raise
4465             self.failUnlessReallyEqual(len(units), 5+1)
4466             # should be parent-first
4467             u0 = units[0]
4468             self.failUnlessEqual(u0["path"], [])
4469             self.failUnlessEqual(u0["type"], "directory")
4470             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4471             u0cr = u0["check-results"]
4472             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4473
4474             ugood = [u for u in units
4475                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4476             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4477             ugoodcr = ugood["check-results"]
4478             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4479
4480             stats = units[-1]
4481             self.failUnlessEqual(stats["type"], "stats")
4482             s = stats["stats"]
4483             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4484             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4485             self.failUnlessReallyEqual(s["count-directories"], 1)
4486             self.failUnlessReallyEqual(s["count-unknown"], 1)
4487         d.addCallback(_done)
4488
4489         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4490         def _check_manifest(res):
4491             self.failUnless(res.endswith("\n"))
4492             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4493             self.failUnlessReallyEqual(len(units), 5+1)
4494             self.failUnlessEqual(units[-1]["type"], "stats")
4495             first = units[0]
4496             self.failUnlessEqual(first["path"], [])
4497             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4498             self.failUnlessEqual(first["type"], "directory")
4499             stats = units[-1]["stats"]
4500             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4501             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4502             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4503             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4504             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4505         d.addCallback(_check_manifest)
4506
4507         # now add root/subdir and root/subdir/grandchild, then make subdir
4508         # unrecoverable, then see what happens
4509
4510         d.addCallback(lambda ign:
4511                       self.rootnode.create_subdirectory(u"subdir"))
4512         d.addCallback(_stash_uri, "subdir")
4513         d.addCallback(lambda subdir_node:
4514                       subdir_node.add_file(u"grandchild",
4515                                            upload.Data(DATA+"2",
4516                                                        convergence="")))
4517         d.addCallback(_stash_uri, "grandchild")
4518
4519         d.addCallback(lambda ign:
4520                       self.delete_shares_numbered(self.uris["subdir"],
4521                                                   range(1, 10)))
4522
4523         # root
4524         # root/good
4525         # root/small
4526         # root/sick
4527         # root/future
4528         # root/subdir [unrecoverable]
4529         # root/subdir/grandchild
4530
4531         # how should a streaming-JSON API indicate fatal error?
4532         # answer: emit ERROR: instead of a JSON string
4533
4534         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4535         def _check_broken_manifest(res):
4536             lines = res.splitlines()
4537             error_lines = [i
4538                            for (i,line) in enumerate(lines)
4539                            if line.startswith("ERROR:")]
4540             if not error_lines:
4541                 self.fail("no ERROR: in output: %s" % (res,))
4542             first_error = error_lines[0]
4543             error_line = lines[first_error]
4544             error_msg = lines[first_error+1:]
4545             error_msg_s = "\n".join(error_msg) + "\n"
4546             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4547                               error_line)
4548             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4549             units = [simplejson.loads(line) for line in lines[:first_error]]
4550             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4551             last_unit = units[-1]
4552             self.failUnlessEqual(last_unit["path"], ["subdir"])
4553         d.addCallback(_check_broken_manifest)
4554
4555         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4556         def _check_broken_deepcheck(res):
4557             lines = res.splitlines()
4558             error_lines = [i
4559                            for (i,line) in enumerate(lines)
4560                            if line.startswith("ERROR:")]
4561             if not error_lines:
4562                 self.fail("no ERROR: in output: %s" % (res,))
4563             first_error = error_lines[0]
4564             error_line = lines[first_error]
4565             error_msg = lines[first_error+1:]
4566             error_msg_s = "\n".join(error_msg) + "\n"
4567             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4568                               error_line)
4569             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4570             units = [simplejson.loads(line) for line in lines[:first_error]]
4571             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4572             last_unit = units[-1]
4573             self.failUnlessEqual(last_unit["path"], ["subdir"])
4574             r = last_unit["check-results"]["results"]
4575             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4576             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4577             self.failUnlessReallyEqual(r["recoverable"], False)
4578         d.addCallback(_check_broken_deepcheck)
4579
4580         d.addErrback(self.explain_web_error)
4581         return d
4582
4583     def test_deep_check_and_repair(self):
4584         self.basedir = "web/Grid/deep_check_and_repair"
4585         self.set_up_grid()
4586         c0 = self.g.clients[0]
4587         self.uris = {}
4588         self.fileurls = {}
4589         DATA = "data" * 100
4590         d = c0.create_dirnode()
4591         def _stash_root_and_create_file(n):
4592             self.rootnode = n
4593             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4594             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4595         d.addCallback(_stash_root_and_create_file)
4596         def _stash_uri(fn, which):
4597             self.uris[which] = fn.get_uri()
4598         d.addCallback(_stash_uri, "good")
4599         d.addCallback(lambda ign:
4600                       self.rootnode.add_file(u"small",
4601                                              upload.Data("literal",
4602                                                         convergence="")))
4603         d.addCallback(_stash_uri, "small")
4604         d.addCallback(lambda ign:
4605                       self.rootnode.add_file(u"sick",
4606                                              upload.Data(DATA+"1",
4607                                                         convergence="")))
4608         d.addCallback(_stash_uri, "sick")
4609         #d.addCallback(lambda ign:
4610         #              self.rootnode.add_file(u"dead",
4611         #                                     upload.Data(DATA+"2",
4612         #                                                convergence="")))
4613         #d.addCallback(_stash_uri, "dead")
4614
4615         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4616         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4617         #d.addCallback(_stash_uri, "corrupt")
4618
4619         def _clobber_shares(ignored):
4620             good_shares = self.find_uri_shares(self.uris["good"])
4621             self.failUnlessReallyEqual(len(good_shares), 10)
4622             sick_shares = self.find_uri_shares(self.uris["sick"])
4623             os.unlink(sick_shares[0][2])
4624             #dead_shares = self.find_uri_shares(self.uris["dead"])
4625             #for i in range(1, 10):
4626             #    os.unlink(dead_shares[i][2])
4627
4628             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4629             #cso = CorruptShareOptions()
4630             #cso.stdout = StringIO()
4631             #cso.parseOptions([c_shares[0][2]])
4632             #corrupt_share(cso)
4633         d.addCallback(_clobber_shares)
4634
4635         # root
4636         # root/good   CHK, 10 shares
4637         # root/small  LIT
4638         # root/sick   CHK, 9 shares
4639
4640         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4641         def _done(res):
4642             units = [simplejson.loads(line)
4643                      for line in res.splitlines()
4644                      if line]
4645             self.failUnlessReallyEqual(len(units), 4+1)
4646             # should be parent-first
4647             u0 = units[0]
4648             self.failUnlessEqual(u0["path"], [])
4649             self.failUnlessEqual(u0["type"], "directory")
4650             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4651             u0crr = u0["check-and-repair-results"]
4652             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4653             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4654
4655             ugood = [u for u in units
4656                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4657             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4658             ugoodcrr = ugood["check-and-repair-results"]
4659             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4660             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4661
4662             usick = [u for u in units
4663                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4664             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4665             usickcrr = usick["check-and-repair-results"]
4666             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4667             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4668             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4669             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4670
4671             stats = units[-1]
4672             self.failUnlessEqual(stats["type"], "stats")
4673             s = stats["stats"]
4674             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4675             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4676             self.failUnlessReallyEqual(s["count-directories"], 1)
4677         d.addCallback(_done)
4678
4679         d.addErrback(self.explain_web_error)
4680         return d
4681
4682     def _count_leases(self, ignored, which):
4683         u = self.uris[which]
4684         shares = self.find_uri_shares(u)
4685         lease_counts = []
4686         for shnum, serverid, fn in shares:
4687             sf = get_share_file(fn)
4688             num_leases = len(list(sf.get_leases()))
4689             lease_counts.append( (fn, num_leases) )
4690         return lease_counts
4691
4692     def _assert_leasecount(self, lease_counts, expected):
4693         for (fn, num_leases) in lease_counts:
4694             if num_leases != expected:
4695                 self.fail("expected %d leases, have %d, on %s" %
4696                           (expected, num_leases, fn))
4697
4698     def test_add_lease(self):
4699         self.basedir = "web/Grid/add_lease"
4700         self.set_up_grid(num_clients=2)
4701         c0 = self.g.clients[0]
4702         self.uris = {}
4703         DATA = "data" * 100
4704         d = c0.upload(upload.Data(DATA, convergence=""))
4705         def _stash_uri(ur, which):
4706             self.uris[which] = ur.uri
4707         d.addCallback(_stash_uri, "one")
4708         d.addCallback(lambda ign:
4709                       c0.upload(upload.Data(DATA+"1", convergence="")))
4710         d.addCallback(_stash_uri, "two")
4711         def _stash_mutable_uri(n, which):
4712             self.uris[which] = n.get_uri()
4713             assert isinstance(self.uris[which], str)
4714         d.addCallback(lambda ign:
4715             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4716         d.addCallback(_stash_mutable_uri, "mutable")
4717
4718         def _compute_fileurls(ignored):
4719             self.fileurls = {}
4720             for which in self.uris:
4721                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4722         d.addCallback(_compute_fileurls)
4723
4724         d.addCallback(self._count_leases, "one")
4725         d.addCallback(self._assert_leasecount, 1)
4726         d.addCallback(self._count_leases, "two")
4727         d.addCallback(self._assert_leasecount, 1)
4728         d.addCallback(self._count_leases, "mutable")
4729         d.addCallback(self._assert_leasecount, 1)
4730
4731         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4732         def _got_html_good(res):
4733             self.failUnlessIn("Healthy", res)
4734             self.failIfIn("Not Healthy", res)
4735         d.addCallback(_got_html_good)
4736
4737         d.addCallback(self._count_leases, "one")
4738         d.addCallback(self._assert_leasecount, 1)
4739         d.addCallback(self._count_leases, "two")
4740         d.addCallback(self._assert_leasecount, 1)
4741         d.addCallback(self._count_leases, "mutable")
4742         d.addCallback(self._assert_leasecount, 1)
4743
4744         # this CHECK uses the original client, which uses the same
4745         # lease-secrets, so it will just renew the original lease
4746         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4747         d.addCallback(_got_html_good)
4748
4749         d.addCallback(self._count_leases, "one")
4750         d.addCallback(self._assert_leasecount, 1)
4751         d.addCallback(self._count_leases, "two")
4752         d.addCallback(self._assert_leasecount, 1)
4753         d.addCallback(self._count_leases, "mutable")
4754         d.addCallback(self._assert_leasecount, 1)
4755
4756         # this CHECK uses an alternate client, which adds a second lease
4757         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4758         d.addCallback(_got_html_good)
4759
4760         d.addCallback(self._count_leases, "one")
4761         d.addCallback(self._assert_leasecount, 2)
4762         d.addCallback(self._count_leases, "two")
4763         d.addCallback(self._assert_leasecount, 1)
4764         d.addCallback(self._count_leases, "mutable")
4765         d.addCallback(self._assert_leasecount, 1)
4766
4767         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4768         d.addCallback(_got_html_good)
4769
4770         d.addCallback(self._count_leases, "one")
4771         d.addCallback(self._assert_leasecount, 2)
4772         d.addCallback(self._count_leases, "two")
4773         d.addCallback(self._assert_leasecount, 1)
4774         d.addCallback(self._count_leases, "mutable")
4775         d.addCallback(self._assert_leasecount, 1)
4776
4777         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4778                       clientnum=1)
4779         d.addCallback(_got_html_good)
4780
4781         d.addCallback(self._count_leases, "one")
4782         d.addCallback(self._assert_leasecount, 2)
4783         d.addCallback(self._count_leases, "two")
4784         d.addCallback(self._assert_leasecount, 1)
4785         d.addCallback(self._count_leases, "mutable")
4786         d.addCallback(self._assert_leasecount, 2)
4787
4788         d.addErrback(self.explain_web_error)
4789         return d
4790
4791     def test_deep_add_lease(self):
4792         self.basedir = "web/Grid/deep_add_lease"
4793         self.set_up_grid(num_clients=2)
4794         c0 = self.g.clients[0]
4795         self.uris = {}
4796         self.fileurls = {}
4797         DATA = "data" * 100
4798         d = c0.create_dirnode()
4799         def _stash_root_and_create_file(n):
4800             self.rootnode = n
4801             self.uris["root"] = n.get_uri()
4802             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4803             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4804         d.addCallback(_stash_root_and_create_file)
4805         def _stash_uri(fn, which):
4806             self.uris[which] = fn.get_uri()
4807         d.addCallback(_stash_uri, "one")
4808         d.addCallback(lambda ign:
4809                       self.rootnode.add_file(u"small",
4810                                              upload.Data("literal",
4811                                                         convergence="")))
4812         d.addCallback(_stash_uri, "small")
4813
4814         d.addCallback(lambda ign:
4815             c0.create_mutable_file(publish.MutableData("mutable")))
4816         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4817         d.addCallback(_stash_uri, "mutable")
4818
4819         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4820         def _done(res):
4821             units = [simplejson.loads(line)
4822                      for line in res.splitlines()
4823                      if line]
4824             # root, one, small, mutable,   stats
4825             self.failUnlessReallyEqual(len(units), 4+1)
4826         d.addCallback(_done)
4827
4828         d.addCallback(self._count_leases, "root")
4829         d.addCallback(self._assert_leasecount, 1)
4830         d.addCallback(self._count_leases, "one")
4831         d.addCallback(self._assert_leasecount, 1)
4832         d.addCallback(self._count_leases, "mutable")
4833         d.addCallback(self._assert_leasecount, 1)
4834
4835         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4836         d.addCallback(_done)
4837
4838         d.addCallback(self._count_leases, "root")
4839         d.addCallback(self._assert_leasecount, 1)
4840         d.addCallback(self._count_leases, "one")
4841         d.addCallback(self._assert_leasecount, 1)
4842         d.addCallback(self._count_leases, "mutable")
4843         d.addCallback(self._assert_leasecount, 1)
4844
4845         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4846                       clientnum=1)
4847         d.addCallback(_done)
4848
4849         d.addCallback(self._count_leases, "root")
4850         d.addCallback(self._assert_leasecount, 2)
4851         d.addCallback(self._count_leases, "one")
4852         d.addCallback(self._assert_leasecount, 2)
4853         d.addCallback(self._count_leases, "mutable")
4854         d.addCallback(self._assert_leasecount, 2)
4855
4856         d.addErrback(self.explain_web_error)
4857         return d
4858
4859
4860     def test_exceptions(self):
4861         self.basedir = "web/Grid/exceptions"
4862         self.set_up_grid(num_clients=1, num_servers=2)
4863         c0 = self.g.clients[0]
4864         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4865         self.fileurls = {}
4866         DATA = "data" * 100
4867         d = c0.create_dirnode()
4868         def _stash_root(n):
4869             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4870             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4871             return n
4872         d.addCallback(_stash_root)
4873         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4874         def _stash_bad(ur):
4875             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4876             self.delete_shares_numbered(ur.uri, range(1,10))
4877
4878             u = uri.from_string(ur.uri)
4879             u.key = testutil.flip_bit(u.key, 0)
4880             baduri = u.to_string()
4881             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4882         d.addCallback(_stash_bad)
4883         d.addCallback(lambda ign: c0.create_dirnode())
4884         def _mangle_dirnode_1share(n):
4885             u = n.get_uri()
4886             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4887             self.fileurls["dir-1share-json"] = url + "?t=json"
4888             self.delete_shares_numbered(u, range(1,10))
4889         d.addCallback(_mangle_dirnode_1share)
4890         d.addCallback(lambda ign: c0.create_dirnode())
4891         def _mangle_dirnode_0share(n):
4892             u = n.get_uri()
4893             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4894             self.fileurls["dir-0share-json"] = url + "?t=json"
4895             self.delete_shares_numbered(u, range(0,10))
4896         d.addCallback(_mangle_dirnode_0share)
4897
4898         # NotEnoughSharesError should be reported sensibly, with a
4899         # text/plain explanation of the problem, and perhaps some
4900         # information on which shares *could* be found.
4901
4902         d.addCallback(lambda ignored:
4903                       self.shouldHTTPError("GET unrecoverable",
4904                                            410, "Gone", "NoSharesError",
4905                                            self.GET, self.fileurls["0shares"]))
4906         def _check_zero_shares(body):
4907             self.failIfIn("<html>", body)
4908             body = " ".join(body.strip().split())
4909             exp = ("NoSharesError: no shares could be found. "
4910                    "Zero shares usually indicates a corrupt URI, or that "
4911                    "no servers were connected, but it might also indicate "
4912                    "severe corruption. You should perform a filecheck on "
4913                    "this object to learn more. The full error message is: "
4914                    "no shares (need 3). Last failure: None")
4915             self.failUnlessReallyEqual(exp, body)
4916         d.addCallback(_check_zero_shares)
4917
4918
4919         d.addCallback(lambda ignored:
4920                       self.shouldHTTPError("GET 1share",
4921                                            410, "Gone", "NotEnoughSharesError",
4922                                            self.GET, self.fileurls["1share"]))
4923         def _check_one_share(body):
4924             self.failIfIn("<html>", body)
4925             body = " ".join(body.strip().split())
4926             msgbase = ("NotEnoughSharesError: This indicates that some "
4927                        "servers were unavailable, or that shares have been "
4928                        "lost to server departure, hard drive failure, or disk "
4929                        "corruption. You should perform a filecheck on "
4930                        "this object to learn more. The full error message is:"
4931                        )
4932             msg1 = msgbase + (" ran out of shares:"
4933                               " complete=sh0"
4934                               " pending="
4935                               " overdue= unused= need 3. Last failure: None")
4936             msg2 = msgbase + (" ran out of shares:"
4937                               " complete="
4938                               " pending=Share(sh0-on-xgru5)"
4939                               " overdue= unused= need 3. Last failure: None")
4940             self.failUnless(body == msg1 or body == msg2, body)
4941         d.addCallback(_check_one_share)
4942
4943         d.addCallback(lambda ignored:
4944                       self.shouldHTTPError("GET imaginary",
4945                                            404, "Not Found", None,
4946                                            self.GET, self.fileurls["imaginary"]))
4947         def _missing_child(body):
4948             self.failUnlessIn("No such child: imaginary", body)
4949         d.addCallback(_missing_child)
4950
4951         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4952         def _check_0shares_dir_html(body):
4953             self.failUnlessIn("<html>", body)
4954             # we should see the regular page, but without the child table or
4955             # the dirops forms
4956             body = " ".join(body.strip().split())
4957             self.failUnlessIn('href="?t=info">More info on this directory',
4958                               body)
4959             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4960                    "could not be retrieved, because there were insufficient "
4961                    "good shares. This might indicate that no servers were "
4962                    "connected, insufficient servers were connected, the URI "
4963                    "was corrupt, or that shares have been lost due to server "
4964                    "departure, hard drive failure, or disk corruption. You "
4965                    "should perform a filecheck on this object to learn more.")
4966             self.failUnlessIn(exp, body)
4967             self.failUnlessIn("No upload forms: directory is unreadable", body)
4968         d.addCallback(_check_0shares_dir_html)
4969
4970         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4971         def _check_1shares_dir_html(body):
4972             # at some point, we'll split UnrecoverableFileError into 0-shares
4973             # and some-shares like we did for immutable files (since there
4974             # are different sorts of advice to offer in each case). For now,
4975             # they present the same way.
4976             self.failUnlessIn("<html>", body)
4977             body = " ".join(body.strip().split())
4978             self.failUnlessIn('href="?t=info">More info on this directory',
4979                               body)
4980             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4981                    "could not be retrieved, because there were insufficient "
4982                    "good shares. This might indicate that no servers were "
4983                    "connected, insufficient servers were connected, the URI "
4984                    "was corrupt, or that shares have been lost due to server "
4985                    "departure, hard drive failure, or disk corruption. You "
4986                    "should perform a filecheck on this object to learn more.")
4987             self.failUnlessIn(exp, body)
4988             self.failUnlessIn("No upload forms: directory is unreadable", body)
4989         d.addCallback(_check_1shares_dir_html)
4990
4991         d.addCallback(lambda ignored:
4992                       self.shouldHTTPError("GET dir-0share-json",
4993                                            410, "Gone", "UnrecoverableFileError",
4994                                            self.GET,
4995                                            self.fileurls["dir-0share-json"]))
4996         def _check_unrecoverable_file(body):
4997             self.failIfIn("<html>", body)
4998             body = " ".join(body.strip().split())
4999             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5000                    "could not be retrieved, because there were insufficient "
5001                    "good shares. This might indicate that no servers were "
5002                    "connected, insufficient servers were connected, the URI "
5003                    "was corrupt, or that shares have been lost due to server "
5004                    "departure, hard drive failure, or disk corruption. You "
5005                    "should perform a filecheck on this object to learn more.")
5006             self.failUnlessReallyEqual(exp, body)
5007         d.addCallback(_check_unrecoverable_file)
5008
5009         d.addCallback(lambda ignored:
5010                       self.shouldHTTPError("GET dir-1share-json",
5011                                            410, "Gone", "UnrecoverableFileError",
5012                                            self.GET,
5013                                            self.fileurls["dir-1share-json"]))
5014         d.addCallback(_check_unrecoverable_file)
5015
5016         d.addCallback(lambda ignored:
5017                       self.shouldHTTPError("GET imaginary",
5018                                            404, "Not Found", None,
5019                                            self.GET, self.fileurls["imaginary"]))
5020
5021         # attach a webapi child that throws a random error, to test how it
5022         # gets rendered.
5023         w = c0.getServiceNamed("webish")
5024         w.root.putChild("ERRORBOOM", ErrorBoom())
5025
5026         # "Accept: */*" :        should get a text/html stack trace
5027         # "Accept: text/plain" : should get a text/plain stack trace
5028         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5029         # no Accept header:      should get a text/html stack trace
5030
5031         d.addCallback(lambda ignored:
5032                       self.shouldHTTPError("GET errorboom_html",
5033                                            500, "Internal Server Error", None,
5034                                            self.GET, "ERRORBOOM",
5035                                            headers={"accept": ["*/*"]}))
5036         def _internal_error_html1(body):
5037             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5038         d.addCallback(_internal_error_html1)
5039
5040         d.addCallback(lambda ignored:
5041                       self.shouldHTTPError("GET errorboom_text",
5042                                            500, "Internal Server Error", None,
5043                                            self.GET, "ERRORBOOM",
5044                                            headers={"accept": ["text/plain"]}))
5045         def _internal_error_text2(body):
5046             self.failIfIn("<html>", body)
5047             self.failUnless(body.startswith("Traceback "), body)
5048         d.addCallback(_internal_error_text2)
5049
5050         CLI_accepts = "text/plain, application/octet-stream"
5051         d.addCallback(lambda ignored:
5052                       self.shouldHTTPError("GET errorboom_text",
5053                                            500, "Internal Server Error", None,
5054                                            self.GET, "ERRORBOOM",
5055                                            headers={"accept": [CLI_accepts]}))
5056         def _internal_error_text3(body):
5057             self.failIfIn("<html>", body)
5058             self.failUnless(body.startswith("Traceback "), body)
5059         d.addCallback(_internal_error_text3)
5060
5061         d.addCallback(lambda ignored:
5062                       self.shouldHTTPError("GET errorboom_text",
5063                                            500, "Internal Server Error", None,
5064                                            self.GET, "ERRORBOOM"))
5065         def _internal_error_html4(body):
5066             self.failUnlessIn("<html>", body)
5067         d.addCallback(_internal_error_html4)
5068
5069         def _flush_errors(res):
5070             # Trial: please ignore the CompletelyUnhandledError in the logs
5071             self.flushLoggedErrors(CompletelyUnhandledError)
5072             return res
5073         d.addBoth(_flush_errors)
5074
5075         return d
5076
5077     def test_blacklist(self):
5078         # download from a blacklisted URI, get an error
5079         self.basedir = "web/Grid/blacklist"
5080         self.set_up_grid()
5081         c0 = self.g.clients[0]
5082         c0_basedir = c0.basedir
5083         fn = os.path.join(c0_basedir, "access.blacklist")
5084         self.uris = {}
5085         DATA = "off-limits " * 50
5086
5087         d = c0.upload(upload.Data(DATA, convergence=""))
5088         def _stash_uri_and_create_dir(ur):
5089             self.uri = ur.uri
5090             self.url = "uri/"+self.uri
5091             u = uri.from_string_filenode(self.uri)
5092             self.si = u.get_storage_index()
5093             childnode = c0.create_node_from_uri(self.uri, None)
5094             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5095         d.addCallback(_stash_uri_and_create_dir)
5096         def _stash_dir(node):
5097             self.dir_node = node
5098             self.dir_uri = node.get_uri()
5099             self.dir_url = "uri/"+self.dir_uri
5100         d.addCallback(_stash_dir)
5101         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5102         def _check_dir_html(body):
5103             self.failUnlessIn("<html>", body)
5104             self.failUnlessIn("blacklisted.txt</a>", body)
5105         d.addCallback(_check_dir_html)
5106         d.addCallback(lambda ign: self.GET(self.url))
5107         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5108
5109         def _blacklist(ign):
5110             f = open(fn, "w")
5111             f.write(" # this is a comment\n")
5112             f.write(" \n")
5113             f.write("\n") # also exercise blank lines
5114             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5115             f.close()
5116             # clients should be checking the blacklist each time, so we don't
5117             # need to restart the client
5118         d.addCallback(_blacklist)
5119         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5120                                                        403, "Forbidden",
5121                                                        "Access Prohibited: off-limits",
5122                                                        self.GET, self.url))
5123
5124         # We should still be able to list the parent directory, in HTML...
5125         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5126         def _check_dir_html2(body):
5127             self.failUnlessIn("<html>", body)
5128             self.failUnlessIn("blacklisted.txt</strike>", body)
5129         d.addCallback(_check_dir_html2)
5130
5131         # ... and in JSON (used by CLI).
5132         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5133         def _check_dir_json(res):
5134             data = simplejson.loads(res)
5135             self.failUnless(isinstance(data, list), data)
5136             self.failUnlessEqual(data[0], "dirnode")
5137             self.failUnless(isinstance(data[1], dict), data)
5138             self.failUnlessIn("children", data[1])
5139             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5140             childdata = data[1]["children"]["blacklisted.txt"]
5141             self.failUnless(isinstance(childdata, list), data)
5142             self.failUnlessEqual(childdata[0], "filenode")
5143             self.failUnless(isinstance(childdata[1], dict), data)
5144         d.addCallback(_check_dir_json)
5145
5146         def _unblacklist(ign):
5147             open(fn, "w").close()
5148             # the Blacklist object watches mtime to tell when the file has
5149             # changed, but on windows this test will run faster than the
5150             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5151             # to force a reload.
5152             self.g.clients[0].blacklist.last_mtime -= 2.0
5153         d.addCallback(_unblacklist)
5154
5155         # now a read should work
5156         d.addCallback(lambda ign: self.GET(self.url))
5157         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5158
5159         # read again to exercise the blacklist-is-unchanged logic
5160         d.addCallback(lambda ign: self.GET(self.url))
5161         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5162
5163         # now add a blacklisted directory, and make sure files under it are
5164         # refused too
5165         def _add_dir(ign):
5166             childnode = c0.create_node_from_uri(self.uri, None)
5167             return c0.create_dirnode({u"child": (childnode,{}) })
5168         d.addCallback(_add_dir)
5169         def _get_dircap(dn):
5170             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5171             self.dir_url_base = "uri/"+dn.get_write_uri()
5172             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5173             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5174             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5175             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5176         d.addCallback(_get_dircap)
5177         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5178         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5179         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5180         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5181         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5182         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5183         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5184         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5185         d.addCallback(lambda ign: self.GET(self.child_url))
5186         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5187
5188         def _block_dir(ign):
5189             f = open(fn, "w")
5190             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5191             f.close()
5192             self.g.clients[0].blacklist.last_mtime -= 2.0
5193         d.addCallback(_block_dir)
5194         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5195                                                        403, "Forbidden",
5196                                                        "Access Prohibited: dir-off-limits",
5197                                                        self.GET, self.dir_url_base))
5198         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5199                                                        403, "Forbidden",
5200                                                        "Access Prohibited: dir-off-limits",
5201                                                        self.GET, self.dir_url_json1))
5202         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5203                                                        403, "Forbidden",
5204                                                        "Access Prohibited: dir-off-limits",
5205                                                        self.GET, self.dir_url_json2))
5206         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5207                                                        403, "Forbidden",
5208                                                        "Access Prohibited: dir-off-limits",
5209                                                        self.GET, self.dir_url_json_ro))
5210         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5211                                                        403, "Forbidden",
5212                                                        "Access Prohibited: dir-off-limits",
5213                                                        self.GET, self.child_url))
5214         return d
5215
5216
5217 class CompletelyUnhandledError(Exception):
5218     pass
5219 class ErrorBoom(rend.Page):
5220     def beforeRender(self, ctx):
5221         raise CompletelyUnhandledError("whoops")