src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53 DIR_HTML_TAG = '<html lang="en">'
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194     def get_available_space(self):
 195         return 0
 196
 197 class FakeBucketCounter(object):
 198     def get_state(self):
 199         return {"last-complete-bucket-count": 0}
 200     def get_progress(self):
 201         return {"estimated-time-per-cycle": 0,
 202                 "cycle-in-progress": False,
 203                 "remaining-wait-time": 0}
 204
 205 class FakeLeaseChecker(object):
 206     def __init__(self):
 207         self.expiration_enabled = False
 208         self.mode = "age"
 209         self.override_lease_duration = None
 210         self.sharetypes_to_expire = {}
 211     def get_state(self):
 212         return {"history": None}
 213     def get_progress(self):
 214         return {"estimated-time-per-cycle": 0,
 215                 "cycle-in-progress": False,
 216                 "remaining-wait-time": 0}
 217
 218 class FakeStorageServer(service.MultiService):
 219     name = 'storage'
 220     def __init__(self, nodeid, nickname):
 221         service.MultiService.__init__(self)
 222         self.my_nodeid = nodeid
 223         self.nickname = nickname
 224         self.bucket_counter = FakeBucketCounter()
 225         self.lease_checker = FakeLeaseChecker()
 226     def get_stats(self):
 227         return {"storage_server.accepting_immutable_shares": False}
 228
 229 class FakeClient(Client):
 230     def __init__(self):
 231         # don't upcall to Client.__init__, since we only want to initialize a
 232         # minimal subset
 233         service.MultiService.__init__(self)
 234         self.all_contents = {}
 235         self.nodeid = "fake_nodeid"
 236         self.nickname = u"fake_nickname \u263A"
 237         self.introducer_furl = "None"
 238         self.stats_provider = FakeStatsProvider()
 239         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 240         self.helper = None
 241         self.convergence = "some random string"
 242         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 243         # fake knowledge of another server
 244         self.storage_broker.test_add_server("other_nodeid",
 245                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 246         self.introducer_client = None
 247         self.history = FakeHistory()
 248         self.uploader = FakeUploader()
 249         self.uploader.all_contents = self.all_contents
 250         self.uploader.setServiceParent(self)
 251         self.blacklist = None
 252         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 253                                        self.uploader, None,
 254                                        None, None, None)
 255         self.nodemaker.all_contents = self.all_contents
 256         self.mutable_file_default = SDMF_VERSION
 257         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 258
 259     def get_long_nodeid(self):
 260         return "v0-nodeid"
 261     def get_long_tubid(self):
 262         return "tubid"
 263
 264     def startService(self):
 265         return service.MultiService.startService(self)
 266     def stopService(self):
 267         return service.MultiService.stopService(self)
 268
 269     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 270
 271 class WebMixin(object):
 272     def setUp(self):
 273         self.s = FakeClient()
 274         self.s.startService()
 275         self.staticdir = self.mktemp()
 276         self.clock = Clock()
 277         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 278                                       clock=self.clock)
 279         self.ws.setServiceParent(self.s)
 280         self.webish_port = self.ws.getPortnum()
 281         self.webish_url = self.ws.getURL()
 282         assert self.webish_url.endswith("/")
 283         self.webish_url = self.webish_url[:-1] # these tests add their own /
 284
 285         l = [ self.s.create_dirnode() for x in range(6) ]
 286         d = defer.DeferredList(l)
 287         def _then(res):
 288             self.public_root = res[0][1]
 289             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 290             self.public_url = "/uri/" + self.public_root.get_uri()
 291             self.private_root = res[1][1]
 292
 293             foo = res[2][1]
 294             self._foo_node = foo
 295             self._foo_uri = foo.get_uri()
 296             self._foo_readonly_uri = foo.get_readonly_uri()
 297             self._foo_verifycap = foo.get_verify_cap().to_string()
 298             # NOTE: we ignore the deferred on all set_uri() calls, because we
 299             # know the fake nodes do these synchronously
 300             self.public_root.set_uri(u"foo", foo.get_uri(),
 301                                      foo.get_readonly_uri())
 302
 303             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 304             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 305             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 306
 307             # sdmf
 308             # XXX: Do we ever use this?
 309             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 310
 311             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 312
 313             # mdmf
 314             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 315             assert self._quux_txt_uri.startswith("URI:MDMF")
 316             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 317
 318             foo.set_uri(u"empty", res[3][1].get_uri(),
 319                         res[3][1].get_readonly_uri())
 320             sub_uri = res[4][1].get_uri()
 321             self._sub_uri = sub_uri
 322             foo.set_uri(u"sub", sub_uri, sub_uri)
 323             sub = self.s.create_node_from_uri(sub_uri)
 324             self._sub_node = sub
 325
 326             _ign, n, blocking_uri = self.makefile(1)
 327             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 328
 329             # filenode to test for html encoding issues
 330             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 331             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 332             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 333             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 334             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 335             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 336             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 337             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 338
 339             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 340             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 341             # still think of it as an umlaut
 342             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 343
 344             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 345             self._baz_file_uri = baz_file
 346             sub.set_uri(u"baz.txt", baz_file, baz_file)
 347
 348             _ign, n, self._bad_file_uri = self.makefile(3)
 349             # this uri should not be downloadable
 350             del self.s.all_contents[self._bad_file_uri]
 351
 352             rodir = res[5][1]
 353             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 354                                      rodir.get_readonly_uri())
 355             rodir.set_uri(u"nor", baz_file, baz_file)
 356
 357             # public/
 358             # public/foo/
 359             # public/foo/bar.txt
 360             # public/foo/baz.txt
 361             # public/foo/quux.txt
 362             # public/foo/blockingfile
 363             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 364             # public/foo/empty/
 365             # public/foo/sub/
 366             # public/foo/sub/baz.txt
 367             # public/reedownlee/
 368             # public/reedownlee/nor
 369             self.NEWFILE_CONTENTS = "newfile contents\n"
 370
 371             return foo.get_metadata_for(u"bar.txt")
 372         d.addCallback(_then)
 373         def _got_metadata(metadata):
 374             self._bar_txt_metadata = metadata
 375         d.addCallback(_got_metadata)
 376         return d
 377
 378     def get_all_contents(self):
 379         return self.s.all_contents
 380
 381     def makefile(self, number):
 382         contents = "contents of file %s\n" % number
 383         n = create_chk_filenode(contents, self.get_all_contents())
 384         return contents, n, n.get_uri()
 385
 386     def makefile_mutable(self, number, mdmf=False):
 387         contents = "contents of mutable file %s\n" % number
 388         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 389         return contents, n, n.get_uri(), n.get_readonly_uri()
 390
 391     def tearDown(self):
 392         return self.s.stopService()
 393
 394     def failUnlessIsBarDotTxt(self, res):
 395         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 396
 397     def failUnlessIsQuuxDotTxt(self, res):
 398         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 399
 400     def failUnlessIsBazDotTxt(self, res):
 401         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 402
 403     def failUnlessIsSubBazDotTxt(self, res):
 404         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 405
 406     def failUnlessIsBarJSON(self, res):
 407         data = simplejson.loads(res)
 408         self.failUnless(isinstance(data, list))
 409         self.failUnlessEqual(data[0], "filenode")
 410         self.failUnless(isinstance(data[1], dict))
 411         self.failIf(data[1]["mutable"])
 412         self.failIfIn("rw_uri", data[1]) # immutable
 413         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 414         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 415         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 416
 417     def failUnlessIsQuuxJSON(self, res, readonly=False):
 418         data = simplejson.loads(res)
 419         self.failUnless(isinstance(data, list))
 420         self.failUnlessEqual(data[0], "filenode")
 421         self.failUnless(isinstance(data[1], dict))
 422         metadata = data[1]
 423         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 424
 425     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 426         self.failUnless(metadata['mutable'])
 427         if readonly:
 428             self.failIfIn("rw_uri", metadata)
 429         else:
 430             self.failUnlessIn("rw_uri", metadata)
 431             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 432         self.failUnlessIn("ro_uri", metadata)
 433         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 434         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 435
 436     def failUnlessIsFooJSON(self, res):
 437         data = simplejson.loads(res)
 438         self.failUnless(isinstance(data, list))
 439         self.failUnlessEqual(data[0], "dirnode", res)
 440         self.failUnless(isinstance(data[1], dict))
 441         self.failUnless(data[1]["mutable"])
 442         self.failUnlessIn("rw_uri", data[1]) # mutable
 443         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 444         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 445         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 446
 447         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 448         self.failUnlessEqual(kidnames,
 449                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 450                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 451         kids = dict( [(unicode(name),value)
 452                       for (name,value)
 453                       in data[1]["children"].iteritems()] )
 454         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 455         self.failUnlessIn("metadata", kids[u"sub"][1])
 456         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 457         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 458         self.failUnlessIn("linkcrtime", tahoe_md)
 459         self.failUnlessIn("linkmotime", tahoe_md)
 460         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 461         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 462         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 463         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 464                                    self._bar_txt_verifycap)
 465         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 466         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 467         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 468                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 469         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 470                                    self._bar_txt_uri)
 471         self.failUnlessIn("quux.txt", kids)
 472         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 473                                    self._quux_txt_uri)
 474         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 475                                    self._quux_txt_readonly_uri)
 476
 477     def GET(self, urlpath, followRedirect=False, return_response=False,
 478             **kwargs):
 479         # if return_response=True, this fires with (data, statuscode,
 480         # respheaders) instead of just data.
 481         assert not isinstance(urlpath, unicode)
 482         url = self.webish_url + urlpath
 483         factory = HTTPClientGETFactory(url, method="GET",
 484                                        followRedirect=followRedirect, **kwargs)
 485         reactor.connectTCP("localhost", self.webish_port, factory)
 486         d = factory.deferred
 487         def _got_data(data):
 488             return (data, factory.status, factory.response_headers)
 489         if return_response:
 490             d.addCallback(_got_data)
 491         return factory.deferred
 492
 493     def HEAD(self, urlpath, return_response=False, **kwargs):
 494         # this requires some surgery, because twisted.web.client doesn't want
 495         # to give us back the response headers.
 496         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 497         reactor.connectTCP("localhost", self.webish_port, factory)
 498         d = factory.deferred
 499         def _got_data(data):
 500             return (data, factory.status, factory.response_headers)
 501         if return_response:
 502             d.addCallback(_got_data)
 503         return factory.deferred
 504
 505     def PUT(self, urlpath, data, **kwargs):
 506         url = self.webish_url + urlpath
 507         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 508
 509     def DELETE(self, urlpath):
 510         url = self.webish_url + urlpath
 511         return client.getPage(url, method="DELETE")
 512
 513     def POST(self, urlpath, followRedirect=False, **fields):
 514         sepbase = "boogabooga"
 515         sep = "--" + sepbase
 516         form = []
 517         form.append(sep)
 518         form.append('Content-Disposition: form-data; name="_charset"')
 519         form.append('')
 520         form.append('UTF-8')
 521         form.append(sep)
 522         for name, value in fields.iteritems():
 523             if isinstance(value, tuple):
 524                 filename, value = value
 525                 form.append('Content-Disposition: form-data; name="%s"; '
 526                             'filename="%s"' % (name, filename.encode("utf-8")))
 527             else:
 528                 form.append('Content-Disposition: form-data; name="%s"' % name)
 529             form.append('')
 530             if isinstance(value, unicode):
 531                 value = value.encode("utf-8")
 532             else:
 533                 value = str(value)
 534             assert isinstance(value, str)
 535             form.append(value)
 536             form.append(sep)
 537         form[-1] += "--"
 538         body = ""
 539         headers = {}
 540         if fields:
 541             body = "\r\n".join(form) + "\r\n"
 542             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 543         return self.POST2(urlpath, body, headers, followRedirect)
 544
 545     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 546         url = self.webish_url + urlpath
 547         return client.getPage(url, method="POST", postdata=body,
 548                               headers=headers, followRedirect=followRedirect)
 549
 550     def shouldFail(self, res, expected_failure, which,
 551                    substring=None, response_substring=None):
 552         if isinstance(res, failure.Failure):
 553             res.trap(expected_failure)
 554             if substring:
 555                 self.failUnlessIn(substring, str(res), which)
 556             if response_substring:
 557                 self.failUnlessIn(response_substring, res.value.response, which)
 558         else:
 559             self.fail("%s was supposed to raise %s, not get '%s'" %
 560                       (which, expected_failure, res))
 561
 562     def shouldFail2(self, expected_failure, which, substring,
 563                     response_substring,
 564                     callable, *args, **kwargs):
 565         assert substring is None or isinstance(substring, str)
 566         assert response_substring is None or isinstance(response_substring, str)
 567         d = defer.maybeDeferred(callable, *args, **kwargs)
 568         def done(res):
 569             if isinstance(res, failure.Failure):
 570                 res.trap(expected_failure)
 571                 if substring:
 572                     self.failUnlessIn(substring, str(res),
 573                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 574                                       (substring, str(res),
 575                                        getattr(res.value, "response", ""),
 576                                        which))
 577                 if response_substring:
 578                     self.failUnlessIn(response_substring, res.value.response,
 579                                       "'%s' not in '%s' for test '%s'" % \
 580                                       (response_substring, res.value.response,
 581                                        which))
 582             else:
 583                 self.fail("%s was supposed to raise %s, not get '%s'" %
 584                           (which, expected_failure, res))
 585         d.addBoth(done)
 586         return d
 587
 588     def should404(self, res, which):
 589         if isinstance(res, failure.Failure):
 590             res.trap(error.Error)
 591             self.failUnlessReallyEqual(res.value.status, "404")
 592         else:
 593             self.fail("%s was supposed to Error(404), not get '%s'" %
 594                       (which, res))
 595
 596     def should302(self, res, which):
 597         if isinstance(res, failure.Failure):
 598             res.trap(error.Error)
 599             self.failUnlessReallyEqual(res.value.status, "302")
 600         else:
 601             self.fail("%s was supposed to Error(302), not get '%s'" %
 602                       (which, res))
 603
 604
 605 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 606     def test_create(self):
 607         pass
 608
 609     def test_welcome(self):
 610         d = self.GET("/")
 611         def _check(res):
 612             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 613             self.failUnlessIn(FAVICON_MARKUP, res)
 614             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 615             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 616             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 617             self.failUnlessIn('Page rendered at', res)
 618             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 619             res_u = res.decode('utf-8')
 620             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 621             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 622             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 623
 624             self.s.basedir = 'web/test_welcome'
 625             fileutil.make_dirs("web/test_welcome")
 626             fileutil.make_dirs("web/test_welcome/private")
 627             return self.GET("/")
 628         d.addCallback(_check)
 629         return d
 630
 631     def test_introducer_status(self):
 632         class MockIntroducerClient(object):
 633             def __init__(self, connected):
 634                 self.connected = connected
 635             def connected_to_introducer(self):
 636                 return self.connected
 637
 638         d = defer.succeed(None)
 639
 640         # introducer not connected, unguessable furl
 641         def _set_introducer_not_connected_unguessable(ign):
 642             self.s.introducer_furl = "pb://someIntroducer/secret"
 643             self.s.introducer_client = MockIntroducerClient(False)
 644             return self.GET("/")
 645         d.addCallback(_set_introducer_not_connected_unguessable)
 646         def _check_introducer_not_connected_unguessable(res):
 647             html = res.replace('\n', ' ')
 648             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 649             self.failIfIn('pb://someIntroducer/secret', html)
 650             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 651         d.addCallback(_check_introducer_not_connected_unguessable)
 652
 653         # introducer connected, unguessable furl
 654         def _set_introducer_connected_unguessable(ign):
 655             self.s.introducer_furl = "pb://someIntroducer/secret"
 656             self.s.introducer_client = MockIntroducerClient(True)
 657             return self.GET("/")
 658         d.addCallback(_set_introducer_connected_unguessable)
 659         def _check_introducer_connected_unguessable(res):
 660             html = res.replace('\n', ' ')
 661             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 662             self.failIfIn('pb://someIntroducer/secret', html)
 663             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 664         d.addCallback(_check_introducer_connected_unguessable)
 665
 666         # introducer connected, guessable furl
 667         def _set_introducer_connected_guessable(ign):
 668             self.s.introducer_furl = "pb://someIntroducer/introducer"
 669             self.s.introducer_client = MockIntroducerClient(True)
 670             return self.GET("/")
 671         d.addCallback(_set_introducer_connected_guessable)
 672         def _check_introducer_connected_guessable(res):
 673             html = res.replace('\n', ' ')
 674             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 675             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 676         d.addCallback(_check_introducer_connected_guessable)
 677         return d
 678
 679     def test_helper_status(self):
 680         d = defer.succeed(None)
 681
 682         # set helper furl to None
 683         def _set_no_helper(ign):
 684             self.s.uploader.helper_furl = None
 685             return self.GET("/")
 686         d.addCallback(_set_no_helper)
 687         def _check_no_helper(res):
 688             html = res.replace('\n', ' ')
 689             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 690         d.addCallback(_check_no_helper)
 691
 692         # enable helper, not connected
 693         def _set_helper_not_connected(ign):
 694             self.s.uploader.helper_furl = "pb://someHelper/secret"
 695             self.s.uploader.helper_connected = False
 696             return self.GET("/")
 697         d.addCallback(_set_helper_not_connected)
 698         def _check_helper_not_connected(res):
 699             html = res.replace('\n', ' ')
 700             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 701             self.failIfIn('pb://someHelper/secret', html)
 702             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 703         d.addCallback(_check_helper_not_connected)
 704
 705         # enable helper, connected
 706         def _set_helper_connected(ign):
 707             self.s.uploader.helper_furl = "pb://someHelper/secret"
 708             self.s.uploader.helper_connected = True
 709             return self.GET("/")
 710         d.addCallback(_set_helper_connected)
 711         def _check_helper_connected(res):
 712             html = res.replace('\n', ' ')
 713             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 714             self.failIfIn('pb://someHelper/secret', html)
 715             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 716         d.addCallback(_check_helper_connected)
 717         return d
 718
 719     def test_storage(self):
 720         d = self.GET("/storage")
 721         def _check(res):
 722             self.failUnlessIn('Storage Server Status', res)
 723             self.failUnlessIn(FAVICON_MARKUP, res)
 724             res_u = res.decode('utf-8')
 725             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 726         d.addCallback(_check)
 727         return d
 728
 729     def test_status(self):
 730         h = self.s.get_history()
 731         dl_num = h.list_all_download_statuses()[0].get_counter()
 732         ul_num = h.list_all_upload_statuses()[0].get_counter()
 733         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 734         pub_num = h.list_all_publish_statuses()[0].get_counter()
 735         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 736         d = self.GET("/status", followRedirect=True)
 737         def _check(res):
 738             self.failUnlessIn('Recent and Active Operations', res)
 739             self.failUnlessIn('"down-%d"' % dl_num, res)
 740             self.failUnlessIn('"up-%d"' % ul_num, res)
 741             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 742             self.failUnlessIn('"publish-%d"' % pub_num, res)
 743             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 744         d.addCallback(_check)
 745         d.addCallback(lambda res: self.GET("/status/?t=json"))
 746         def _check_json(res):
 747             data = simplejson.loads(res)
 748             self.failUnless(isinstance(data, dict))
 749             #active = data["active"]
 750             # TODO: test more. We need a way to fake an active operation
 751             # here.
 752         d.addCallback(_check_json)
 753
 754         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 755         def _check_dl(res):
 756             self.failUnlessIn("File Download Status", res)
 757         d.addCallback(_check_dl)
 758         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 759         def _check_dl_json(res):
 760             data = simplejson.loads(res)
 761             self.failUnless(isinstance(data, dict))
 762             self.failUnlessIn("read", data)
 763             self.failUnlessEqual(data["read"][0]["length"], 120)
 764             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 765             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 766             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 767             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 768             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 769             # serverids[] keys are strings, since that's what JSON does, but
 770             # we'd really like them to be ints
 771             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 772             self.failUnless(data["serverids"].has_key("1"),
 773                             str(data["serverids"]))
 774             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 775                                  str(data["serverids"]))
 776             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 777                                  "phwrsjte")
 778             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 779                                  "cmpuvkjm")
 780             self.failUnlessIn("dyhb", data)
 781             self.failUnlessIn("misc", data)
 782         d.addCallback(_check_dl_json)
 783         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 784         def _check_ul(res):
 785             self.failUnlessIn("File Upload Status", res)
 786         d.addCallback(_check_ul)
 787         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 788         def _check_mapupdate(res):
 789             self.failUnlessIn("Mutable File Servermap Update Status", res)
 790         d.addCallback(_check_mapupdate)
 791         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 792         def _check_publish(res):
 793             self.failUnlessIn("Mutable File Publish Status", res)
 794         d.addCallback(_check_publish)
 795         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 796         def _check_retrieve(res):
 797             self.failUnlessIn("Mutable File Retrieve Status", res)
 798         d.addCallback(_check_retrieve)
 799
 800         return d
 801
 802     def test_status_numbers(self):
 803         drrm = status.DownloadResultsRendererMixin()
 804         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 805         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 806         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 807         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 808         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 809         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 810         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 811         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 812         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 813
 814         urrm = status.UploadResultsRendererMixin()
 815         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 816         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 817         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 818         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 819         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 820         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 821         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 822         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 823         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 824
 825     def test_GET_FILEURL(self):
 826         d = self.GET(self.public_url + "/foo/bar.txt")
 827         d.addCallback(self.failUnlessIsBarDotTxt)
 828         return d
 829
 830     def test_GET_FILEURL_range(self):
 831         headers = {"range": "bytes=1-10"}
 832         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 833                      return_response=True)
 834         def _got((res, status, headers)):
 835             self.failUnlessReallyEqual(int(status), 206)
 836             self.failUnless(headers.has_key("content-range"))
 837             self.failUnlessReallyEqual(headers["content-range"][0],
 838                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 839             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 840         d.addCallback(_got)
 841         return d
 842
 843     def test_GET_FILEURL_partial_range(self):
 844         headers = {"range": "bytes=5-"}
 845         length  = len(self.BAR_CONTENTS)
 846         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 847                      return_response=True)
 848         def _got((res, status, headers)):
 849             self.failUnlessReallyEqual(int(status), 206)
 850             self.failUnless(headers.has_key("content-range"))
 851             self.failUnlessReallyEqual(headers["content-range"][0],
 852                                        "bytes 5-%d/%d" % (length-1, length))
 853             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 854         d.addCallback(_got)
 855         return d
 856
 857     def test_GET_FILEURL_partial_end_range(self):
 858         headers = {"range": "bytes=-5"}
 859         length  = len(self.BAR_CONTENTS)
 860         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 861                      return_response=True)
 862         def _got((res, status, headers)):
 863             self.failUnlessReallyEqual(int(status), 206)
 864             self.failUnless(headers.has_key("content-range"))
 865             self.failUnlessReallyEqual(headers["content-range"][0],
 866                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 867             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 868         d.addCallback(_got)
 869         return d
 870
 871     def test_GET_FILEURL_partial_range_overrun(self):
 872         headers = {"range": "bytes=100-200"}
 873         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 874                              "416 Requested Range not satisfiable",
 875                              "First beyond end of file",
 876                              self.GET, self.public_url + "/foo/bar.txt",
 877                              headers=headers)
 878         return d
 879
 880     def test_HEAD_FILEURL_range(self):
 881         headers = {"range": "bytes=1-10"}
 882         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 883                      return_response=True)
 884         def _got((res, status, headers)):
 885             self.failUnlessReallyEqual(res, "")
 886             self.failUnlessReallyEqual(int(status), 206)
 887             self.failUnless(headers.has_key("content-range"))
 888             self.failUnlessReallyEqual(headers["content-range"][0],
 889                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 890         d.addCallback(_got)
 891         return d
 892
 893     def test_HEAD_FILEURL_partial_range(self):
 894         headers = {"range": "bytes=5-"}
 895         length  = len(self.BAR_CONTENTS)
 896         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 897                      return_response=True)
 898         def _got((res, status, headers)):
 899             self.failUnlessReallyEqual(int(status), 206)
 900             self.failUnless(headers.has_key("content-range"))
 901             self.failUnlessReallyEqual(headers["content-range"][0],
 902                                        "bytes 5-%d/%d" % (length-1, length))
 903         d.addCallback(_got)
 904         return d
 905
 906     def test_HEAD_FILEURL_partial_end_range(self):
 907         headers = {"range": "bytes=-5"}
 908         length  = len(self.BAR_CONTENTS)
 909         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 910                      return_response=True)
 911         def _got((res, status, headers)):
 912             self.failUnlessReallyEqual(int(status), 206)
 913             self.failUnless(headers.has_key("content-range"))
 914             self.failUnlessReallyEqual(headers["content-range"][0],
 915                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 916         d.addCallback(_got)
 917         return d
 918
 919     def test_HEAD_FILEURL_partial_range_overrun(self):
 920         headers = {"range": "bytes=100-200"}
 921         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 922                              "416 Requested Range not satisfiable",
 923                              "",
 924                              self.HEAD, self.public_url + "/foo/bar.txt",
 925                              headers=headers)
 926         return d
 927
 928     def test_GET_FILEURL_range_bad(self):
 929         headers = {"range": "BOGUS=fizbop-quarnak"}
 930         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 931                      return_response=True)
 932         def _got((res, status, headers)):
 933             self.failUnlessReallyEqual(int(status), 200)
 934             self.failUnless(not headers.has_key("content-range"))
 935             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 936         d.addCallback(_got)
 937         return d
 938
 939     def test_HEAD_FILEURL(self):
 940         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 941         def _got((res, status, headers)):
 942             self.failUnlessReallyEqual(res, "")
 943             self.failUnlessReallyEqual(headers["content-length"][0],
 944                                        str(len(self.BAR_CONTENTS)))
 945             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 946         d.addCallback(_got)
 947         return d
 948
 949     def test_GET_FILEURL_named(self):
 950         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 951         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 952         d = self.GET(base + "/@@name=/blah.txt")
 953         d.addCallback(self.failUnlessIsBarDotTxt)
 954         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 955         d.addCallback(self.failUnlessIsBarDotTxt)
 956         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 957         d.addCallback(self.failUnlessIsBarDotTxt)
 958         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 959         d.addCallback(self.failUnlessIsBarDotTxt)
 960         save_url = base + "?save=true&filename=blah.txt"
 961         d.addCallback(lambda res: self.GET(save_url))
 962         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 963         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 964         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 965         u_url = base + "?save=true&filename=" + u_fn_e
 966         d.addCallback(lambda res: self.GET(u_url))
 967         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 968         return d
 969
 970     def test_PUT_FILEURL_named_bad(self):
 971         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 972         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 973                              "400 Bad Request",
 974                              "/file can only be used with GET or HEAD",
 975                              self.PUT, base + "/@@name=/blah.txt", "")
 976         return d
 977
 978
 979     def test_GET_DIRURL_named_bad(self):
 980         base = "/file/%s" % urllib.quote(self._foo_uri)
 981         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 982                              "400 Bad Request",
 983                              "is not a file-cap",
 984                              self.GET, base + "/@@name=/blah.txt")
 985         return d
 986
 987     def test_GET_slash_file_bad(self):
 988         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 989                              "404 Not Found",
 990                              "/file must be followed by a file-cap and a name",
 991                              self.GET, "/file")
 992         return d
 993
 994     def test_GET_unhandled_URI_named(self):
 995         contents, n, newuri = self.makefile(12)
 996         verifier_cap = n.get_verify_cap().to_string()
 997         base = "/file/%s" % urllib.quote(verifier_cap)
 998         # client.create_node_from_uri() can't handle verify-caps
 999         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1000                              "400 Bad Request", "is not a file-cap",
1001                              self.GET, base)
1002         return d
1003
1004     def test_GET_unhandled_URI(self):
1005         contents, n, newuri = self.makefile(12)
1006         verifier_cap = n.get_verify_cap().to_string()
1007         base = "/uri/%s" % urllib.quote(verifier_cap)
1008         # client.create_node_from_uri() can't handle verify-caps
1009         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1010                              "400 Bad Request",
1011                              "GET unknown URI type: can only do t=info",
1012                              self.GET, base)
1013         return d
1014
1015     def test_GET_FILE_URI(self):
1016         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1017         d = self.GET(base)
1018         d.addCallback(self.failUnlessIsBarDotTxt)
1019         return d
1020
1021     def test_GET_FILE_URI_mdmf(self):
1022         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1023         d = self.GET(base)
1024         d.addCallback(self.failUnlessIsQuuxDotTxt)
1025         return d
1026
1027     def test_GET_FILE_URI_mdmf_extensions(self):
1028         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1029         d = self.GET(base)
1030         d.addCallback(self.failUnlessIsQuuxDotTxt)
1031         return d
1032
1033     def test_GET_FILE_URI_mdmf_readonly(self):
1034         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1035         d = self.GET(base)
1036         d.addCallback(self.failUnlessIsQuuxDotTxt)
1037         return d
1038
1039     def test_GET_FILE_URI_badchild(self):
1040         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1041         errmsg = "Files have no children, certainly not named 'boguschild'"
1042         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1043                              "400 Bad Request", errmsg,
1044                              self.GET, base)
1045         return d
1046
1047     def test_PUT_FILE_URI_badchild(self):
1048         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1049         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1050         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1051                              "400 Bad Request", errmsg,
1052                              self.PUT, base, "")
1053         return d
1054
1055     def test_PUT_FILE_URI_mdmf(self):
1056         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1057         self._quux_new_contents = "new_contents"
1058         d = self.GET(base)
1059         d.addCallback(lambda res:
1060             self.failUnlessIsQuuxDotTxt(res))
1061         d.addCallback(lambda ignored:
1062             self.PUT(base, self._quux_new_contents))
1063         d.addCallback(lambda ignored:
1064             self.GET(base))
1065         d.addCallback(lambda res:
1066             self.failUnlessReallyEqual(res, self._quux_new_contents))
1067         return d
1068
1069     def test_PUT_FILE_URI_mdmf_extensions(self):
1070         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1071         self._quux_new_contents = "new_contents"
1072         d = self.GET(base)
1073         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1074         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1075         d.addCallback(lambda ignored: self.GET(base))
1076         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1077                                                        res))
1078         return d
1079
1080     def test_PUT_FILE_URI_mdmf_readonly(self):
1081         # We're not allowed to PUT things to a readonly cap.
1082         base = "/uri/%s" % self._quux_txt_readonly_uri
1083         d = self.GET(base)
1084         d.addCallback(lambda res:
1085             self.failUnlessIsQuuxDotTxt(res))
1086         # What should we get here? We get a 500 error now; that's not right.
1087         d.addCallback(lambda ignored:
1088             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1089                              "400 Bad Request", "read-only cap",
1090                              self.PUT, base, "new data"))
1091         return d
1092
1093     def test_PUT_FILE_URI_sdmf_readonly(self):
1094         # We're not allowed to put things to a readonly cap.
1095         base = "/uri/%s" % self._baz_txt_readonly_uri
1096         d = self.GET(base)
1097         d.addCallback(lambda res:
1098             self.failUnlessIsBazDotTxt(res))
1099         d.addCallback(lambda ignored:
1100             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1101                              "400 Bad Request", "read-only cap",
1102                              self.PUT, base, "new_data"))
1103         return d
1104
1105     def test_GET_etags(self):
1106
1107         def _check_etags(uri):
1108             d1 = _get_etag(uri)
1109             d2 = _get_etag(uri, 'json')
1110             d = defer.DeferredList([d1, d2], consumeErrors=True)
1111             def _check(results):
1112                 # All deferred must succeed
1113                 self.failUnless(all([r[0] for r in results]))
1114                 # the etag for the t=json form should be just like the etag
1115                 # fo the default t='' form, but with a 'json' suffix
1116                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1117             d.addCallback(_check)
1118             return d
1119
1120         def _get_etag(uri, t=''):
1121             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1122             d = self.GET(targetbase, return_response=True, followRedirect=True)
1123             def _just_the_etag(result):
1124                 data, response, headers = result
1125                 etag = headers['etag'][0]
1126                 if uri.startswith('URI:DIR'):
1127                     self.failUnless(etag.startswith('DIR:'), etag)
1128                 return etag
1129             return d.addCallback(_just_the_etag)
1130
1131         # Check that etags work with immutable directories
1132         (newkids, caps) = self._create_immutable_children()
1133         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1134                       simplejson.dumps(newkids))
1135         def _stash_immdir_uri(uri):
1136             self._immdir_uri = uri
1137             return uri
1138         d.addCallback(_stash_immdir_uri)
1139         d.addCallback(_check_etags)
1140
1141         # Check that etags work with immutable files
1142         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1143
1144         # use the ETag on GET
1145         def _check_match(ign):
1146             uri = "/uri/%s" % self._bar_txt_uri
1147             d = self.GET(uri, return_response=True)
1148             # extract the ETag
1149             d.addCallback(lambda (data, code, headers):
1150                           headers['etag'][0])
1151             # do a GET that's supposed to match the ETag
1152             d.addCallback(lambda etag:
1153                           self.GET(uri, return_response=True,
1154                                    headers={"If-None-Match": etag}))
1155             # make sure it short-circuited (304 instead of 200)
1156             d.addCallback(lambda (data, code, headers):
1157                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1158             return d
1159         d.addCallback(_check_match)
1160
1161         def _no_etag(uri, t):
1162             target = "/uri/%s?t=%s" % (uri, t)
1163             d = self.GET(target, return_response=True, followRedirect=True)
1164             d.addCallback(lambda (data, code, headers):
1165                           self.failIf("etag" in headers, target))
1166             return d
1167         def _yes_etag(uri, t):
1168             target = "/uri/%s?t=%s" % (uri, t)
1169             d = self.GET(target, return_response=True, followRedirect=True)
1170             d.addCallback(lambda (data, code, headers):
1171                           self.failUnless("etag" in headers, target))
1172             return d
1173
1174         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1175         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1176         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1177         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1178         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1179
1180         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1181         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1182         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1183         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1184         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1185         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1186
1187         return d
1188
1189     # TODO: version of this with a Unicode filename
1190     def test_GET_FILEURL_save(self):
1191         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1192                      return_response=True)
1193         def _got((res, statuscode, headers)):
1194             content_disposition = headers["content-disposition"][0]
1195             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1196             self.failUnlessIsBarDotTxt(res)
1197         d.addCallback(_got)
1198         return d
1199
1200     def test_GET_FILEURL_missing(self):
1201         d = self.GET(self.public_url + "/foo/missing")
1202         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1203         return d
1204
1205     def test_GET_FILEURL_info_mdmf(self):
1206         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1207         def _got(res):
1208             self.failUnlessIn("mutable file (mdmf)", res)
1209             self.failUnlessIn(self._quux_txt_uri, res)
1210             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1211         d.addCallback(_got)
1212         return d
1213
1214     def test_GET_FILEURL_info_mdmf_readonly(self):
1215         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1216         def _got(res):
1217             self.failUnlessIn("mutable file (mdmf)", res)
1218             self.failIfIn(self._quux_txt_uri, res)
1219             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1220         d.addCallback(_got)
1221         return d
1222
1223     def test_GET_FILEURL_info_sdmf(self):
1224         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1225         def _got(res):
1226             self.failUnlessIn("mutable file (sdmf)", res)
1227             self.failUnlessIn(self._baz_txt_uri, res)
1228         d.addCallback(_got)
1229         return d
1230
1231     def test_GET_FILEURL_info_mdmf_extensions(self):
1232         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1233         def _got(res):
1234             self.failUnlessIn("mutable file (mdmf)", res)
1235             self.failUnlessIn(self._quux_txt_uri, res)
1236             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1237         d.addCallback(_got)
1238         return d
1239
1240     def test_PUT_overwrite_only_files(self):
1241         # create a directory, put a file in that directory.
1242         contents, n, filecap = self.makefile(8)
1243         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1244         d.addCallback(lambda res:
1245             self.PUT(self.public_url + "/foo/dir/file1.txt",
1246                      self.NEWFILE_CONTENTS))
1247         # try to overwrite the file with replace=only-files
1248         # (this should work)
1249         d.addCallback(lambda res:
1250             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1251                      filecap))
1252         d.addCallback(lambda res:
1253             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1254                  "There was already a child by that name, and you asked me "
1255                  "to not replace it",
1256                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1257                  filecap))
1258         return d
1259
1260     def test_PUT_NEWFILEURL(self):
1261         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1262         # TODO: we lose the response code, so we can't check this
1263         #self.failUnlessReallyEqual(responsecode, 201)
1264         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1265         d.addCallback(lambda res:
1266                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1267                                                       self.NEWFILE_CONTENTS))
1268         return d
1269
1270     def test_PUT_NEWFILEURL_not_mutable(self):
1271         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1272                      self.NEWFILE_CONTENTS)
1273         # TODO: we lose the response code, so we can't check this
1274         #self.failUnlessReallyEqual(responsecode, 201)
1275         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1276         d.addCallback(lambda res:
1277                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1278                                                       self.NEWFILE_CONTENTS))
1279         return d
1280
1281     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1282         # this should get us a few segments of an MDMF mutable file,
1283         # which we can then test for.
1284         contents = self.NEWFILE_CONTENTS * 300000
1285         d = self.PUT("/uri?format=mdmf",
1286                      contents)
1287         def _got_filecap(filecap):
1288             self.failUnless(filecap.startswith("URI:MDMF"))
1289             return filecap
1290         d.addCallback(_got_filecap)
1291         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1292         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1293         return d
1294
1295     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1296         contents = self.NEWFILE_CONTENTS * 300000
1297         d = self.PUT("/uri?format=sdmf",
1298                      contents)
1299         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1300         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1301         return d
1302
1303     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1304         contents = self.NEWFILE_CONTENTS * 300000
1305         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1306                                     400, "Bad Request", "Unknown format: foo",
1307                                     self.PUT, "/uri?format=foo",
1308                                     contents)
1309
1310     def test_PUT_NEWFILEURL_range_bad(self):
1311         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1312         target = self.public_url + "/foo/new.txt"
1313         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1314                              "501 Not Implemented",
1315                              "Content-Range in PUT not yet supported",
1316                              # (and certainly not for immutable files)
1317                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1318                              headers=headers)
1319         d.addCallback(lambda res:
1320                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1321         return d
1322
1323     def test_PUT_NEWFILEURL_mutable(self):
1324         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1325                      self.NEWFILE_CONTENTS)
1326         # TODO: we lose the response code, so we can't check this
1327         #self.failUnlessReallyEqual(responsecode, 201)
1328         def _check_uri(res):
1329             u = uri.from_string_mutable_filenode(res)
1330             self.failUnless(u.is_mutable())
1331             self.failIf(u.is_readonly())
1332             return res
1333         d.addCallback(_check_uri)
1334         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1335         d.addCallback(lambda res:
1336                       self.failUnlessMutableChildContentsAre(self._foo_node,
1337                                                              u"new.txt",
1338                                                              self.NEWFILE_CONTENTS))
1339         return d
1340
1341     def test_PUT_NEWFILEURL_mutable_toobig(self):
1342         # It is okay to upload large mutable files, so we should be able
1343         # to do that.
1344         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1345                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1346         return d
1347
1348     def test_PUT_NEWFILEURL_replace(self):
1349         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1350         # TODO: we lose the response code, so we can't check this
1351         #self.failUnlessReallyEqual(responsecode, 200)
1352         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1353         d.addCallback(lambda res:
1354                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1355                                                       self.NEWFILE_CONTENTS))
1356         return d
1357
1358     def test_PUT_NEWFILEURL_bad_t(self):
1359         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1360                              "PUT to a file: bad t=bogus",
1361                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1362                              "contents")
1363         return d
1364
1365     def test_PUT_NEWFILEURL_no_replace(self):
1366         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1367                      self.NEWFILE_CONTENTS)
1368         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1369                   "409 Conflict",
1370                   "There was already a child by that name, and you asked me "
1371                   "to not replace it")
1372         return d
1373
1374     def test_PUT_NEWFILEURL_mkdirs(self):
1375         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1376         fn = self._foo_node
1377         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1378         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1379         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1380         d.addCallback(lambda res:
1381                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1382                                                       self.NEWFILE_CONTENTS))
1383         return d
1384
1385     def test_PUT_NEWFILEURL_blocked(self):
1386         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1387                      self.NEWFILE_CONTENTS)
1388         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1389                   "409 Conflict",
1390                   "Unable to create directory 'blockingfile': a file was in the way")
1391         return d
1392
1393     def test_PUT_NEWFILEURL_emptyname(self):
1394         # an empty pathname component (i.e. a double-slash) is disallowed
1395         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1396                              "400 Bad Request",
1397                              "The webapi does not allow empty pathname components",
1398                              self.PUT, self.public_url + "/foo//new.txt", "")
1399         return d
1400
1401     def test_DELETE_FILEURL(self):
1402         d = self.DELETE(self.public_url + "/foo/bar.txt")
1403         d.addCallback(lambda res:
1404                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1405         return d
1406
1407     def test_DELETE_FILEURL_missing(self):
1408         d = self.DELETE(self.public_url + "/foo/missing")
1409         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1410         return d
1411
1412     def test_DELETE_FILEURL_missing2(self):
1413         d = self.DELETE(self.public_url + "/missing/missing")
1414         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1415         return d
1416
1417     def failUnlessHasBarDotTxtMetadata(self, res):
1418         data = simplejson.loads(res)
1419         self.failUnless(isinstance(data, list))
1420         self.failUnlessIn("metadata", data[1])
1421         self.failUnlessIn("tahoe", data[1]["metadata"])
1422         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1423         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1424         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1425                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1426
1427     def test_GET_FILEURL_json(self):
1428         # twisted.web.http.parse_qs ignores any query args without an '=', so
1429         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1430         # instead. This may make it tricky to emulate the S3 interface
1431         # completely.
1432         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1433         def _check1(data):
1434             self.failUnlessIsBarJSON(data)
1435             self.failUnlessHasBarDotTxtMetadata(data)
1436             return
1437         d.addCallback(_check1)
1438         return d
1439
1440     def test_GET_FILEURL_json_mutable_type(self):
1441         # The JSON should include format, which says whether the
1442         # file is SDMF or MDMF
1443         d = self.PUT("/uri?format=mdmf",
1444                      self.NEWFILE_CONTENTS * 300000)
1445         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1446         def _got_json(json, version):
1447             data = simplejson.loads(json)
1448             assert "filenode" == data[0]
1449             data = data[1]
1450             assert isinstance(data, dict)
1451
1452             self.failUnlessIn("format", data)
1453             self.failUnlessEqual(data["format"], version)
1454
1455         d.addCallback(_got_json, "MDMF")
1456         # Now make an SDMF file and check that it is reported correctly.
1457         d.addCallback(lambda ignored:
1458             self.PUT("/uri?format=sdmf",
1459                       self.NEWFILE_CONTENTS * 300000))
1460         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1461         d.addCallback(_got_json, "SDMF")
1462         return d
1463
1464     def test_GET_FILEURL_json_mdmf(self):
1465         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1466         d.addCallback(self.failUnlessIsQuuxJSON)
1467         return d
1468
1469     def test_GET_FILEURL_json_missing(self):
1470         d = self.GET(self.public_url + "/foo/missing?json")
1471         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1472         return d
1473
1474     def test_GET_FILEURL_uri(self):
1475         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1476         def _check(res):
1477             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1478         d.addCallback(_check)
1479         d.addCallback(lambda res:
1480                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1481         def _check2(res):
1482             # for now, for files, uris and readonly-uris are the same
1483             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1484         d.addCallback(_check2)
1485         return d
1486
1487     def test_GET_FILEURL_badtype(self):
1488         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1489                                  "bad t=bogus",
1490                                  self.GET,
1491                                  self.public_url + "/foo/bar.txt?t=bogus")
1492         return d
1493
1494     def test_CSS_FILE(self):
1495         d = self.GET("/tahoe.css", followRedirect=True)
1496         def _check(res):
1497             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1498             self.failUnless(CSS_STYLE.search(res), res)
1499         d.addCallback(_check)
1500         return d
1501
1502     def test_GET_FILEURL_uri_missing(self):
1503         d = self.GET(self.public_url + "/foo/missing?t=uri")
1504         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1505         return d
1506
1507     def _check_upload_and_mkdir_forms(self, html):
1508         # We should have a form to create a file, with radio buttons that allow
1509         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1510         self.failUnlessIn('name="t" value="upload"', html)
1511         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1512         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1513         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1514
1515         # We should also have the ability to create a mutable directory, with
1516         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1517         # or MDMF directory.
1518         self.failUnlessIn('name="t" value="mkdir"', html)
1519         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1520         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1521
1522         self.failUnlessIn(FAVICON_MARKUP, html)
1523
1524     def test_GET_DIRECTORY_html(self):
1525         d = self.GET(self.public_url + "/foo", followRedirect=True)
1526         def _check(html):
1527             self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1528             self._check_upload_and_mkdir_forms(html)
1529             self.failUnlessIn("quux", html)
1530         d.addCallback(_check)
1531         return d
1532
1533     def test_GET_DIRECTORY_html_filenode_encoding(self):
1534         d = self.GET(self.public_url + "/foo", followRedirect=True)
1535         def _check(html):
1536             # Check if encoded entries are there
1537             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1538                               + self._htmlname_escaped + '</a>', html)
1539             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1540             self.failIfIn(self._htmlname_escaped_double, html)
1541             # Make sure that Nevow escaping actually works by checking for unsafe characters
1542             # and that '&' is escaped.
1543             for entity in '<>':
1544                 self.failUnlessIn(entity, self._htmlname_raw)
1545                 self.failIfIn(entity, self._htmlname_escaped)
1546             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1547             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1548         d.addCallback(_check)
1549         return d
1550
1551     def test_GET_root_html(self):
1552         d = self.GET("/")
1553         d.addCallback(self._check_upload_and_mkdir_forms)
1554         return d
1555
1556     def test_GET_DIRURL(self):
1557         # the addSlash means we get a redirect here
1558         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1559         ROOT = "../../.."
1560         d = self.GET(self.public_url + "/foo", followRedirect=True)
1561         def _check(res):
1562             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1563
1564             # the FILE reference points to a URI, but it should end in bar.txt
1565             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1566                        (ROOT, urllib.quote(self._bar_txt_uri)))
1567             get_bar = "".join([r'<td>FILE</td>',
1568                                r'\s+<td>',
1569                                r'<a href="%s">bar.txt</a>' % bar_url,
1570                                r'</td>',
1571                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1572                                ])
1573             self.failUnless(re.search(get_bar, res), res)
1574             for label in ['unlink', 'rename/relink']:
1575                 for line in res.split("\n"):
1576                     # find the line that contains the relevant button for bar.txt
1577                     if ("form action" in line and
1578                         ('value="%s"' % (label,)) in line and
1579                         'value="bar.txt"' in line):
1580                         # the form target should use a relative URL
1581                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1582                         self.failUnlessIn('action="%s"' % foo_url, line)
1583                         # and the when_done= should too
1584                         #done_url = urllib.quote(???)
1585                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1586
1587                         # 'unlink' needs to use POST because it directly has a side effect
1588                         if label == 'unlink':
1589                             self.failUnlessIn('method="post"', line)
1590                         break
1591                 else:
1592                     self.fail("unable to find '%s bar.txt' line" % (label,))
1593
1594             # the DIR reference just points to a URI
1595             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1596             get_sub = ((r'<td>DIR</td>')
1597                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1598             self.failUnless(re.search(get_sub, res), res)
1599         d.addCallback(_check)
1600
1601         # look at a readonly directory
1602         d.addCallback(lambda res:
1603                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1604         def _check2(res):
1605             self.failUnlessIn("(read-only)", res)
1606             self.failIfIn("Upload a file", res)
1607         d.addCallback(_check2)
1608
1609         # and at a directory that contains a readonly directory
1610         d.addCallback(lambda res:
1611                       self.GET(self.public_url, followRedirect=True))
1612         def _check3(res):
1613             self.failUnless(re.search('<td>DIR-RO</td>'
1614                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1615         d.addCallback(_check3)
1616
1617         # and an empty directory
1618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1619         def _check4(res):
1620             self.failUnlessIn("directory is empty", res)
1621             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" class="btn" value="Create" />', re.I)
1622             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1623         d.addCallback(_check4)
1624
1625         # and at a literal directory
1626         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1627         d.addCallback(lambda res:
1628                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1629         def _check5(res):
1630             self.failUnlessIn('(immutable)', res)
1631             self.failUnless(re.search('<td>FILE</td>'
1632                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1633         d.addCallback(_check5)
1634         return d
1635
1636     def test_GET_DIRURL_badtype(self):
1637         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1638                                  400, "Bad Request",
1639                                  "bad t=bogus",
1640                                  self.GET,
1641                                  self.public_url + "/foo?t=bogus")
1642         return d
1643
1644     def test_GET_DIRURL_json(self):
1645         d = self.GET(self.public_url + "/foo?t=json")
1646         d.addCallback(self.failUnlessIsFooJSON)
1647         return d
1648
1649     def test_GET_DIRURL_json_format(self):
1650         d = self.PUT(self.public_url + \
1651                      "/foo/sdmf.txt?format=sdmf",
1652                      self.NEWFILE_CONTENTS * 300000)
1653         d.addCallback(lambda ignored:
1654             self.PUT(self.public_url + \
1655                      "/foo/mdmf.txt?format=mdmf",
1656                      self.NEWFILE_CONTENTS * 300000))
1657         # Now we have an MDMF and SDMF file in the directory. If we GET
1658         # its JSON, we should see their encodings.
1659         d.addCallback(lambda ignored:
1660             self.GET(self.public_url + "/foo?t=json"))
1661         def _got_json(json):
1662             data = simplejson.loads(json)
1663             assert data[0] == "dirnode"
1664
1665             data = data[1]
1666             kids = data['children']
1667
1668             mdmf_data = kids['mdmf.txt'][1]
1669             self.failUnlessIn("format", mdmf_data)
1670             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1671
1672             sdmf_data = kids['sdmf.txt'][1]
1673             self.failUnlessIn("format", sdmf_data)
1674             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1675         d.addCallback(_got_json)
1676         return d
1677
1678
1679     def test_POST_DIRURL_manifest_no_ophandle(self):
1680         d = self.shouldFail2(error.Error,
1681                              "test_POST_DIRURL_manifest_no_ophandle",
1682                              "400 Bad Request",
1683                              "slow operation requires ophandle=",
1684                              self.POST, self.public_url, t="start-manifest")
1685         return d
1686
1687     def test_POST_DIRURL_manifest(self):
1688         d = defer.succeed(None)
1689         def getman(ignored, output):
1690             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1691                           followRedirect=True)
1692             d.addCallback(self.wait_for_operation, "125")
1693             d.addCallback(self.get_operation_results, "125", output)
1694             return d
1695         d.addCallback(getman, None)
1696         def _got_html(manifest):
1697             self.failUnlessIn("Manifest of SI=", manifest)
1698             self.failUnlessIn("<td>sub</td>", manifest)
1699             self.failUnlessIn(self._sub_uri, manifest)
1700             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1701             self.failUnlessIn(FAVICON_MARKUP, manifest)
1702         d.addCallback(_got_html)
1703
1704         # both t=status and unadorned GET should be identical
1705         d.addCallback(lambda res: self.GET("/operations/125"))
1706         d.addCallback(_got_html)
1707
1708         d.addCallback(getman, "html")
1709         d.addCallback(_got_html)
1710         d.addCallback(getman, "text")
1711         def _got_text(manifest):
1712             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1713             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1714         d.addCallback(_got_text)
1715         d.addCallback(getman, "JSON")
1716         def _got_json(res):
1717             data = res["manifest"]
1718             got = {}
1719             for (path_list, cap) in data:
1720                 got[tuple(path_list)] = cap
1721             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1722             self.failUnlessIn((u"sub", u"baz.txt"), got)
1723             self.failUnlessIn("finished", res)
1724             self.failUnlessIn("origin", res)
1725             self.failUnlessIn("storage-index", res)
1726             self.failUnlessIn("verifycaps", res)
1727             self.failUnlessIn("stats", res)
1728         d.addCallback(_got_json)
1729         return d
1730
1731     def test_POST_DIRURL_deepsize_no_ophandle(self):
1732         d = self.shouldFail2(error.Error,
1733                              "test_POST_DIRURL_deepsize_no_ophandle",
1734                              "400 Bad Request",
1735                              "slow operation requires ophandle=",
1736                              self.POST, self.public_url, t="start-deep-size")
1737         return d
1738
1739     def test_POST_DIRURL_deepsize(self):
1740         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1741                       followRedirect=True)
1742         d.addCallback(self.wait_for_operation, "126")
1743         d.addCallback(self.get_operation_results, "126", "json")
1744         def _got_json(data):
1745             self.failUnlessReallyEqual(data["finished"], True)
1746             size = data["size"]
1747             self.failUnless(size > 1000)
1748         d.addCallback(_got_json)
1749         d.addCallback(self.get_operation_results, "126", "text")
1750         def _got_text(res):
1751             mo = re.search(r'^size: (\d+)$', res, re.M)
1752             self.failUnless(mo, res)
1753             size = int(mo.group(1))
1754             # with directories, the size varies.
1755             self.failUnless(size > 1000)
1756         d.addCallback(_got_text)
1757         return d
1758
1759     def test_POST_DIRURL_deepstats_no_ophandle(self):
1760         d = self.shouldFail2(error.Error,
1761                              "test_POST_DIRURL_deepstats_no_ophandle",
1762                              "400 Bad Request",
1763                              "slow operation requires ophandle=",
1764                              self.POST, self.public_url, t="start-deep-stats")
1765         return d
1766
1767     def test_POST_DIRURL_deepstats(self):
1768         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1769                       followRedirect=True)
1770         d.addCallback(self.wait_for_operation, "127")
1771         d.addCallback(self.get_operation_results, "127", "json")
1772         def _got_json(stats):
1773             expected = {"count-immutable-files": 4,
1774                         "count-mutable-files": 2,
1775                         "count-literal-files": 0,
1776                         "count-files": 6,
1777                         "count-directories": 3,
1778                         "size-immutable-files": 76,
1779                         "size-literal-files": 0,
1780                         #"size-directories": 1912, # varies
1781                         #"largest-directory": 1590,
1782                         "largest-directory-children": 8,
1783                         "largest-immutable-file": 19,
1784                         }
1785             for k,v in expected.iteritems():
1786                 self.failUnlessReallyEqual(stats[k], v,
1787                                            "stats[%s] was %s, not %s" %
1788                                            (k, stats[k], v))
1789             self.failUnlessReallyEqual(stats["size-files-histogram"],
1790                                        [ [11, 31, 4] ])
1791         d.addCallback(_got_json)
1792         return d
1793
1794     def test_POST_DIRURL_stream_manifest(self):
1795         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1796         def _check(res):
1797             self.failUnless(res.endswith("\n"))
1798             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1799             self.failUnlessReallyEqual(len(units), 10)
1800             self.failUnlessEqual(units[-1]["type"], "stats")
1801             first = units[0]
1802             self.failUnlessEqual(first["path"], [])
1803             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1804             self.failUnlessEqual(first["type"], "directory")
1805             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1806             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1807             self.failIfEqual(baz["storage-index"], None)
1808             self.failIfEqual(baz["verifycap"], None)
1809             self.failIfEqual(baz["repaircap"], None)
1810             # XXX: Add quux and baz to this test.
1811             return
1812         d.addCallback(_check)
1813         return d
1814
1815     def test_GET_DIRURL_uri(self):
1816         d = self.GET(self.public_url + "/foo?t=uri")
1817         def _check(res):
1818             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1819         d.addCallback(_check)
1820         return d
1821
1822     def test_GET_DIRURL_readonly_uri(self):
1823         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1824         def _check(res):
1825             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1826         d.addCallback(_check)
1827         return d
1828
1829     def test_PUT_NEWDIRURL(self):
1830         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1833         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1834         d.addCallback(self.failUnlessNodeKeysAre, [])
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mdmf(self):
1838         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1839         d.addCallback(lambda res:
1840                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1841         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1842         d.addCallback(lambda node:
1843             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1844         return d
1845
1846     def test_PUT_NEWDIRURL_sdmf(self):
1847         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1848                      "")
1849         d.addCallback(lambda res:
1850                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1851         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1852         d.addCallback(lambda node:
1853             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1854         return d
1855
1856     def test_PUT_NEWDIRURL_bad_format(self):
1857         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1858                                     400, "Bad Request", "Unknown format: foo",
1859                                     self.PUT, self.public_url +
1860                                     "/foo/newdir=?t=mkdir&format=foo", "")
1861
1862     def test_POST_NEWDIRURL(self):
1863         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1864         d.addCallback(lambda res:
1865                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1866         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1867         d.addCallback(self.failUnlessNodeKeysAre, [])
1868         return d
1869
1870     def test_POST_NEWDIRURL_mdmf(self):
1871         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1872         d.addCallback(lambda res:
1873                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1874         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875         d.addCallback(lambda node:
1876             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1877         return d
1878
1879     def test_POST_NEWDIRURL_sdmf(self):
1880         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1881         d.addCallback(lambda res:
1882             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1883         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1884         d.addCallback(lambda node:
1885             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1886         return d
1887
1888     def test_POST_NEWDIRURL_bad_format(self):
1889         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1890                                     400, "Bad Request", "Unknown format: foo",
1891                                     self.POST2, self.public_url + \
1892                                     "/foo/newdir?t=mkdir&format=foo", "")
1893
1894     def test_POST_NEWDIRURL_emptyname(self):
1895         # an empty pathname component (i.e. a double-slash) is disallowed
1896         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1897                              "400 Bad Request",
1898                              "The webapi does not allow empty pathname components, i.e. a double slash",
1899                              self.POST, self.public_url + "//?t=mkdir")
1900         return d
1901
1902     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1903         (newkids, caps) = self._create_initial_children()
1904         query = "/foo/newdir?t=mkdir-with-children"
1905         if version == MDMF_VERSION:
1906             query += "&format=mdmf"
1907         elif version == SDMF_VERSION:
1908             query += "&format=sdmf"
1909         else:
1910             version = SDMF_VERSION # for later
1911         d = self.POST2(self.public_url + query,
1912                        simplejson.dumps(newkids))
1913         def _check(uri):
1914             n = self.s.create_node_from_uri(uri.strip())
1915             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1916             self.failUnlessEqual(n._node.get_version(), version)
1917             d2.addCallback(lambda ign:
1918                            self.failUnlessROChildURIIs(n, u"child-imm",
1919                                                        caps['filecap1']))
1920             d2.addCallback(lambda ign:
1921                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1922                                                        caps['filecap2']))
1923             d2.addCallback(lambda ign:
1924                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1925                                                        caps['filecap3']))
1926             d2.addCallback(lambda ign:
1927                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1928                                                        caps['unknown_rocap']))
1929             d2.addCallback(lambda ign:
1930                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1931                                                        caps['unknown_rwcap']))
1932             d2.addCallback(lambda ign:
1933                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1934                                                        caps['unknown_immcap']))
1935             d2.addCallback(lambda ign:
1936                            self.failUnlessRWChildURIIs(n, u"dirchild",
1937                                                        caps['dircap']))
1938             d2.addCallback(lambda ign:
1939                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1940                                                        caps['litdircap']))
1941             d2.addCallback(lambda ign:
1942                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1943                                                        caps['emptydircap']))
1944             return d2
1945         d.addCallback(_check)
1946         d.addCallback(lambda res:
1947                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1948         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1949         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1950         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1951         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1952         return d
1953
1954     def test_POST_NEWDIRURL_initial_children(self):
1955         return self._do_POST_NEWDIRURL_initial_children_test()
1956
1957     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1958         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1959
1960     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1961         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1962
1963     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1964         (newkids, caps) = self._create_initial_children()
1965         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1966                                     400, "Bad Request", "Unknown format: foo",
1967                                     self.POST2, self.public_url + \
1968                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1969                                     simplejson.dumps(newkids))
1970
1971     def test_POST_NEWDIRURL_immutable(self):
1972         (newkids, caps) = self._create_immutable_children()
1973         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1974                        simplejson.dumps(newkids))
1975         def _check(uri):
1976             n = self.s.create_node_from_uri(uri.strip())
1977             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1978             d2.addCallback(lambda ign:
1979                            self.failUnlessROChildURIIs(n, u"child-imm",
1980                                                        caps['filecap1']))
1981             d2.addCallback(lambda ign:
1982                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1983                                                        caps['unknown_immcap']))
1984             d2.addCallback(lambda ign:
1985                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1986                                                        caps['immdircap']))
1987             d2.addCallback(lambda ign:
1988                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1989                                                        caps['litdircap']))
1990             d2.addCallback(lambda ign:
1991                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1992                                                        caps['emptydircap']))
1993             return d2
1994         d.addCallback(_check)
1995         d.addCallback(lambda res:
1996                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1998         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2007         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2008         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2009         d.addErrback(self.explain_web_error)
2010         return d
2011
2012     def test_POST_NEWDIRURL_immutable_bad(self):
2013         (newkids, caps) = self._create_initial_children()
2014         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2015                              "400 Bad Request",
2016                              "needed to be immutable but was not",
2017                              self.POST2,
2018                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2019                              simplejson.dumps(newkids))
2020         return d
2021
2022     def test_PUT_NEWDIRURL_exists(self):
2023         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2024         d.addCallback(lambda res:
2025                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2026         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2027         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2028         return d
2029
2030     def test_PUT_NEWDIRURL_blocked(self):
2031         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2032                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2033                              self.PUT,
2034                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2035         d.addCallback(lambda res:
2036                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2037         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2038         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2039         return d
2040
2041     def test_PUT_NEWDIRURL_mkdirs(self):
2042         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2043         d.addCallback(lambda res:
2044                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2045         d.addCallback(lambda res:
2046                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2047         d.addCallback(lambda res:
2048                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2049         d.addCallback(self.failUnlessNodeKeysAre, [])
2050         return d
2051
2052     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2053         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2054         d.addCallback(lambda ignored:
2055             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2056         d.addCallback(lambda ignored:
2057             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2058         d.addCallback(lambda ignored:
2059             self._foo_node.get_child_at_path(u"subdir"))
2060         def _got_subdir(subdir):
2061             # XXX: What we want?
2062             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2063             self.failUnlessNodeHasChild(subdir, u"newdir")
2064             return subdir.get_child_at_path(u"newdir")
2065         d.addCallback(_got_subdir)
2066         d.addCallback(lambda newdir:
2067             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2068         return d
2069
2070     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2071         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2072         d.addCallback(lambda ignored:
2073             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2074         d.addCallback(lambda ignored:
2075             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2076         d.addCallback(lambda ignored:
2077             self._foo_node.get_child_at_path(u"subdir"))
2078         def _got_subdir(subdir):
2079             # XXX: What we want?
2080             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2081             self.failUnlessNodeHasChild(subdir, u"newdir")
2082             return subdir.get_child_at_path(u"newdir")
2083         d.addCallback(_got_subdir)
2084         d.addCallback(lambda newdir:
2085             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2086         return d
2087
2088     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2089         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2090                                     400, "Bad Request", "Unknown format: foo",
2091                                     self.PUT, self.public_url + \
2092                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2093                                     "")
2094
2095     def test_DELETE_DIRURL(self):
2096         d = self.DELETE(self.public_url + "/foo")
2097         d.addCallback(lambda res:
2098                       self.failIfNodeHasChild(self.public_root, u"foo"))
2099         return d
2100
2101     def test_DELETE_DIRURL_missing(self):
2102         d = self.DELETE(self.public_url + "/foo/missing")
2103         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2104         d.addCallback(lambda res:
2105                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2106         return d
2107
2108     def test_DELETE_DIRURL_missing2(self):
2109         d = self.DELETE(self.public_url + "/missing")
2110         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2111         return d
2112
2113     def dump_root(self):
2114         print "NODEWALK"
2115         w = webish.DirnodeWalkerMixin()
2116         def visitor(childpath, childnode, metadata):
2117             print childpath
2118         d = w.walk(self.public_root, visitor)
2119         return d
2120
2121     def failUnlessNodeKeysAre(self, node, expected_keys):
2122         for k in expected_keys:
2123             assert isinstance(k, unicode)
2124         d = node.list()
2125         def _check(children):
2126             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2127         d.addCallback(_check)
2128         return d
2129     def failUnlessNodeHasChild(self, node, name):
2130         assert isinstance(name, unicode)
2131         d = node.list()
2132         def _check(children):
2133             self.failUnlessIn(name, children)
2134         d.addCallback(_check)
2135         return d
2136     def failIfNodeHasChild(self, node, name):
2137         assert isinstance(name, unicode)
2138         d = node.list()
2139         def _check(children):
2140             self.failIfIn(name, children)
2141         d.addCallback(_check)
2142         return d
2143
2144     def failUnlessChildContentsAre(self, node, name, expected_contents):
2145         assert isinstance(name, unicode)
2146         d = node.get_child_at_path(name)
2147         d.addCallback(lambda node: download_to_data(node))
2148         def _check(contents):
2149             self.failUnlessReallyEqual(contents, expected_contents)
2150         d.addCallback(_check)
2151         return d
2152
2153     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2154         assert isinstance(name, unicode)
2155         d = node.get_child_at_path(name)
2156         d.addCallback(lambda node: node.download_best_version())
2157         def _check(contents):
2158             self.failUnlessReallyEqual(contents, expected_contents)
2159         d.addCallback(_check)
2160         return d
2161
2162     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2163         assert isinstance(name, unicode)
2164         d = node.get_child_at_path(name)
2165         def _check(child):
2166             self.failUnless(child.is_unknown() or not child.is_readonly())
2167             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2168             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2169             expected_ro_uri = self._make_readonly(expected_uri)
2170             if expected_ro_uri:
2171                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2172         d.addCallback(_check)
2173         return d
2174
2175     def failUnlessROChildURIIs(self, node, name, expected_uri):
2176         assert isinstance(name, unicode)
2177         d = node.get_child_at_path(name)
2178         def _check(child):
2179             self.failUnless(child.is_unknown() or child.is_readonly())
2180             self.failUnlessReallyEqual(child.get_write_uri(), None)
2181             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2182             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2183         d.addCallback(_check)
2184         return d
2185
2186     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2187         assert isinstance(name, unicode)
2188         d = node.get_child_at_path(name)
2189         def _check(child):
2190             self.failUnless(child.is_unknown() or not child.is_readonly())
2191             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2192             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2193             expected_ro_uri = self._make_readonly(got_uri)
2194             if expected_ro_uri:
2195                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2196         d.addCallback(_check)
2197         return d
2198
2199     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2200         assert isinstance(name, unicode)
2201         d = node.get_child_at_path(name)
2202         def _check(child):
2203             self.failUnless(child.is_unknown() or child.is_readonly())
2204             self.failUnlessReallyEqual(child.get_write_uri(), None)
2205             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2206             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2207         d.addCallback(_check)
2208         return d
2209
2210     def failUnlessCHKURIHasContents(self, got_uri, contents):
2211         self.failUnless(self.get_all_contents()[got_uri] == contents)
2212
2213     def test_POST_upload(self):
2214         d = self.POST(self.public_url + "/foo", t="upload",
2215                       file=("new.txt", self.NEWFILE_CONTENTS))
2216         fn = self._foo_node
2217         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2218         d.addCallback(lambda res:
2219                       self.failUnlessChildContentsAre(fn, u"new.txt",
2220                                                       self.NEWFILE_CONTENTS))
2221         return d
2222
2223     def test_POST_upload_unicode(self):
2224         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2225         d = self.POST(self.public_url + "/foo", t="upload",
2226                       file=(filename, self.NEWFILE_CONTENTS))
2227         fn = self._foo_node
2228         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2229         d.addCallback(lambda res:
2230                       self.failUnlessChildContentsAre(fn, filename,
2231                                                       self.NEWFILE_CONTENTS))
2232         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2233         d.addCallback(lambda res: self.GET(target_url))
2234         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2235                                                                   self.NEWFILE_CONTENTS,
2236                                                                   contents))
2237         return d
2238
2239     def test_POST_upload_unicode_named(self):
2240         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2241         d = self.POST(self.public_url + "/foo", t="upload",
2242                       name=filename,
2243                       file=("overridden", self.NEWFILE_CONTENTS))
2244         fn = self._foo_node
2245         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2246         d.addCallback(lambda res:
2247                       self.failUnlessChildContentsAre(fn, filename,
2248                                                       self.NEWFILE_CONTENTS))
2249         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2250         d.addCallback(lambda res: self.GET(target_url))
2251         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2252                                                                   self.NEWFILE_CONTENTS,
2253                                                                   contents))
2254         return d
2255
2256     def test_POST_upload_no_link(self):
2257         d = self.POST("/uri", t="upload",
2258                       file=("new.txt", self.NEWFILE_CONTENTS))
2259         def _check_upload_results(page):
2260             # this should be a page which describes the results of the upload
2261             # that just finished.
2262             self.failUnlessIn("Upload Results:", page)
2263             self.failUnlessIn("URI:", page)
2264             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2265             mo = uri_re.search(page)
2266             self.failUnless(mo, page)
2267             new_uri = mo.group(1)
2268             return new_uri
2269         d.addCallback(_check_upload_results)
2270         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2271         return d
2272
2273     def test_POST_upload_no_link_whendone(self):
2274         d = self.POST("/uri", t="upload", when_done="/",
2275                       file=("new.txt", self.NEWFILE_CONTENTS))
2276         d.addBoth(self.shouldRedirect, "/")
2277         return d
2278
2279     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2280         d = defer.maybeDeferred(callable, *args, **kwargs)
2281         def done(res):
2282             if isinstance(res, failure.Failure):
2283                 res.trap(error.PageRedirect)
2284                 statuscode = res.value.status
2285                 target = res.value.location
2286                 return checker(statuscode, target)
2287             self.fail("%s: callable was supposed to redirect, not return '%s'"
2288                       % (which, res))
2289         d.addBoth(done)
2290         return d
2291
2292     def test_POST_upload_no_link_whendone_results(self):
2293         def check(statuscode, target):
2294             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2295             self.failUnless(target.startswith(self.webish_url), target)
2296             return client.getPage(target, method="GET")
2297         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2298         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2299                                  check,
2300                                  self.POST, "/uri", t="upload",
2301                                  when_done="/%75ri/%(uri)s",
2302                                  file=("new.txt", self.NEWFILE_CONTENTS))
2303         d.addCallback(lambda res:
2304                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2305         return d
2306
2307     def test_POST_upload_no_link_mutable(self):
2308         d = self.POST("/uri", t="upload", mutable="true",
2309                       file=("new.txt", self.NEWFILE_CONTENTS))
2310         def _check(filecap):
2311             filecap = filecap.strip()
2312             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2313             self.filecap = filecap
2314             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2315             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2316             n = self.s.create_node_from_uri(filecap)
2317             return n.download_best_version()
2318         d.addCallback(_check)
2319         def _check2(data):
2320             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2321             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2322         d.addCallback(_check2)
2323         def _check3(data):
2324             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2325             return self.GET("/file/%s" % urllib.quote(self.filecap))
2326         d.addCallback(_check3)
2327         def _check4(data):
2328             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2329         d.addCallback(_check4)
2330         return d
2331
2332     def test_POST_upload_no_link_mutable_toobig(self):
2333         # The SDMF size limit is no longer in place, so we should be
2334         # able to upload mutable files that are as large as we want them
2335         # to be.
2336         d = self.POST("/uri", t="upload", mutable="true",
2337                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2338         return d
2339
2340
2341     def test_POST_upload_format_unlinked(self):
2342         def _check_upload_unlinked(ign, format, uri_prefix):
2343             filename = format + ".txt"
2344             d = self.POST("/uri?t=upload&format=" + format,
2345                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2346             def _got_results(results):
2347                 if format.upper() in ("SDMF", "MDMF"):
2348                     # webapi.rst says this returns a filecap
2349                     filecap = results
2350                 else:
2351                     # for immutable, it returns an "upload results page", and
2352                     # the filecap is buried inside
2353                     line = [l for l in results.split("\n") if "URI: " in l][0]
2354                     mo = re.search(r'<span>([^<]+)</span>', line)
2355                     filecap = mo.group(1)
2356                 self.failUnless(filecap.startswith(uri_prefix),
2357                                 (uri_prefix, filecap))
2358                 return self.GET("/uri/%s?t=json" % filecap)
2359             d.addCallback(_got_results)
2360             def _got_json(json):
2361                 data = simplejson.loads(json)
2362                 data = data[1]
2363                 self.failUnlessIn("format", data)
2364                 self.failUnlessEqual(data["format"], format.upper())
2365             d.addCallback(_got_json)
2366             return d
2367         d = defer.succeed(None)
2368         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2369         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2370         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2371         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2372         return d
2373
2374     def test_POST_upload_bad_format_unlinked(self):
2375         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2376                                     400, "Bad Request", "Unknown format: foo",
2377                                     self.POST,
2378                                     "/uri?t=upload&format=foo",
2379                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2380
2381     def test_POST_upload_format(self):
2382         def _check_upload(ign, format, uri_prefix, fn=None):
2383             filename = format + ".txt"
2384             d = self.POST(self.public_url +
2385                           "/foo?t=upload&format=" + format,
2386                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2387             def _got_filecap(filecap):
2388                 if fn is not None:
2389                     filenameu = unicode(filename)
2390                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2391                 self.failUnless(filecap.startswith(uri_prefix))
2392                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2393             d.addCallback(_got_filecap)
2394             def _got_json(json):
2395                 data = simplejson.loads(json)
2396                 data = data[1]
2397                 self.failUnlessIn("format", data)
2398                 self.failUnlessEqual(data["format"], format.upper())
2399             d.addCallback(_got_json)
2400             return d
2401
2402         d = defer.succeed(None)
2403         d.addCallback(_check_upload, "chk", "URI:CHK")
2404         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2405         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2406         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2407         return d
2408
2409     def test_POST_upload_bad_format(self):
2410         return self.shouldHTTPError("POST_upload_bad_format",
2411                                     400, "Bad Request", "Unknown format: foo",
2412                                     self.POST, self.public_url + \
2413                                     "/foo?t=upload&format=foo",
2414                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2415
2416     def test_POST_upload_mutable(self):
2417         # this creates a mutable file
2418         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2419                       file=("new.txt", self.NEWFILE_CONTENTS))
2420         fn = self._foo_node
2421         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2422         d.addCallback(lambda res:
2423                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2424                                                              self.NEWFILE_CONTENTS))
2425         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2426         def _got(newnode):
2427             self.failUnless(IMutableFileNode.providedBy(newnode))
2428             self.failUnless(newnode.is_mutable())
2429             self.failIf(newnode.is_readonly())
2430             self._mutable_node = newnode
2431             self._mutable_uri = newnode.get_uri()
2432         d.addCallback(_got)
2433
2434         # now upload it again and make sure that the URI doesn't change
2435         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2436         d.addCallback(lambda res:
2437                       self.POST(self.public_url + "/foo", t="upload",
2438                                 mutable="true",
2439                                 file=("new.txt", NEWER_CONTENTS)))
2440         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2441         d.addCallback(lambda res:
2442                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2443                                                              NEWER_CONTENTS))
2444         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2445         def _got2(newnode):
2446             self.failUnless(IMutableFileNode.providedBy(newnode))
2447             self.failUnless(newnode.is_mutable())
2448             self.failIf(newnode.is_readonly())
2449             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2450         d.addCallback(_got2)
2451
2452         # upload a second time, using PUT instead of POST
2453         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2454         d.addCallback(lambda res:
2455                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2456         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2457         d.addCallback(lambda res:
2458                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2459                                                              NEW2_CONTENTS))
2460
2461         # finally list the directory, since mutable files are displayed
2462         # slightly differently
2463
2464         d.addCallback(lambda res:
2465                       self.GET(self.public_url + "/foo/",
2466                                followRedirect=True))
2467         def _check_page(res):
2468             # TODO: assert more about the contents
2469             self.failUnlessIn("SSK", res)
2470             return res
2471         d.addCallback(_check_page)
2472
2473         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2474         def _got3(newnode):
2475             self.failUnless(IMutableFileNode.providedBy(newnode))
2476             self.failUnless(newnode.is_mutable())
2477             self.failIf(newnode.is_readonly())
2478             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2479         d.addCallback(_got3)
2480
2481         # look at the JSON form of the enclosing directory
2482         d.addCallback(lambda res:
2483                       self.GET(self.public_url + "/foo/?t=json",
2484                                followRedirect=True))
2485         def _check_page_json(res):
2486             parsed = simplejson.loads(res)
2487             self.failUnlessEqual(parsed[0], "dirnode")
2488             children = dict( [(unicode(name),value)
2489                               for (name,value)
2490                               in parsed[1]["children"].iteritems()] )
2491             self.failUnlessIn(u"new.txt", children)
2492             new_json = children[u"new.txt"]
2493             self.failUnlessEqual(new_json[0], "filenode")
2494             self.failUnless(new_json[1]["mutable"])
2495             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2496             ro_uri = self._mutable_node.get_readonly().to_string()
2497             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2498         d.addCallback(_check_page_json)
2499
2500         # and the JSON form of the file
2501         d.addCallback(lambda res:
2502                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2503         def _check_file_json(res):
2504             parsed = simplejson.loads(res)
2505             self.failUnlessEqual(parsed[0], "filenode")
2506             self.failUnless(parsed[1]["mutable"])
2507             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2508             ro_uri = self._mutable_node.get_readonly().to_string()
2509             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2510         d.addCallback(_check_file_json)
2511
2512         # and look at t=uri and t=readonly-uri
2513         d.addCallback(lambda res:
2514                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2515         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2516         d.addCallback(lambda res:
2517                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2518         def _check_ro_uri(res):
2519             ro_uri = self._mutable_node.get_readonly().to_string()
2520             self.failUnlessReallyEqual(res, ro_uri)
2521         d.addCallback(_check_ro_uri)
2522
2523         # make sure we can get to it from /uri/URI
2524         d.addCallback(lambda res:
2525                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2526         d.addCallback(lambda res:
2527                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2528
2529         # and that HEAD computes the size correctly
2530         d.addCallback(lambda res:
2531                       self.HEAD(self.public_url + "/foo/new.txt",
2532                                 return_response=True))
2533         def _got_headers((res, status, headers)):
2534             self.failUnlessReallyEqual(res, "")
2535             self.failUnlessReallyEqual(headers["content-length"][0],
2536                                        str(len(NEW2_CONTENTS)))
2537             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2538         d.addCallback(_got_headers)
2539
2540         # make sure that outdated size limits aren't enforced anymore.
2541         d.addCallback(lambda ignored:
2542             self.POST(self.public_url + "/foo", t="upload",
2543                       mutable="true",
2544                       file=("new.txt",
2545                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2546         d.addErrback(self.dump_error)
2547         return d
2548
2549     def test_POST_upload_mutable_toobig(self):
2550         # SDMF had a size limti that was removed a while ago. MDMF has
2551         # never had a size limit. Test to make sure that we do not
2552         # encounter errors when trying to upload large mutable files,
2553         # since there should be no coded prohibitions regarding large
2554         # mutable files.
2555         d = self.POST(self.public_url + "/foo",
2556                       t="upload", mutable="true",
2557                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2558         return d
2559
2560     def dump_error(self, f):
2561         # if the web server returns an error code (like 400 Bad Request),
2562         # web.client.getPage puts the HTTP response body into the .response
2563         # attribute of the exception object that it gives back. It does not
2564         # appear in the Failure's repr(), so the ERROR that trial displays
2565         # will be rather terse and unhelpful. addErrback this method to the
2566         # end of your chain to get more information out of these errors.
2567         if f.check(error.Error):
2568             print "web.error.Error:"
2569             print f
2570             print f.value.response
2571         return f
2572
2573     def test_POST_upload_replace(self):
2574         d = self.POST(self.public_url + "/foo", t="upload",
2575                       file=("bar.txt", self.NEWFILE_CONTENTS))
2576         fn = self._foo_node
2577         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2578         d.addCallback(lambda res:
2579                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2580                                                       self.NEWFILE_CONTENTS))
2581         return d
2582
2583     def test_POST_upload_no_replace_ok(self):
2584         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2585                       file=("new.txt", self.NEWFILE_CONTENTS))
2586         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2587         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2588                                                              self.NEWFILE_CONTENTS))
2589         return d
2590
2591     def test_POST_upload_no_replace_queryarg(self):
2592         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2593                       file=("bar.txt", self.NEWFILE_CONTENTS))
2594         d.addBoth(self.shouldFail, error.Error,
2595                   "POST_upload_no_replace_queryarg",
2596                   "409 Conflict",
2597                   "There was already a child by that name, and you asked me "
2598                   "to not replace it")
2599         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2600         d.addCallback(self.failUnlessIsBarDotTxt)
2601         return d
2602
2603     def test_POST_upload_no_replace_field(self):
2604         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2605                       file=("bar.txt", self.NEWFILE_CONTENTS))
2606         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2607                   "409 Conflict",
2608                   "There was already a child by that name, and you asked me "
2609                   "to not replace it")
2610         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2611         d.addCallback(self.failUnlessIsBarDotTxt)
2612         return d
2613
2614     def test_POST_upload_whendone(self):
2615         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2616                       file=("new.txt", self.NEWFILE_CONTENTS))
2617         d.addBoth(self.shouldRedirect, "/THERE")
2618         fn = self._foo_node
2619         d.addCallback(lambda res:
2620                       self.failUnlessChildContentsAre(fn, u"new.txt",
2621                                                       self.NEWFILE_CONTENTS))
2622         return d
2623
2624     def test_POST_upload_named(self):
2625         fn = self._foo_node
2626         d = self.POST(self.public_url + "/foo", t="upload",
2627                       name="new.txt", file=self.NEWFILE_CONTENTS)
2628         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2629         d.addCallback(lambda res:
2630                       self.failUnlessChildContentsAre(fn, u"new.txt",
2631                                                       self.NEWFILE_CONTENTS))
2632         return d
2633
2634     def test_POST_upload_named_badfilename(self):
2635         d = self.POST(self.public_url + "/foo", t="upload",
2636                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2637         d.addBoth(self.shouldFail, error.Error,
2638                   "test_POST_upload_named_badfilename",
2639                   "400 Bad Request",
2640                   "name= may not contain a slash",
2641                   )
2642         # make sure that nothing was added
2643         d.addCallback(lambda res:
2644                       self.failUnlessNodeKeysAre(self._foo_node,
2645                                                  [self._htmlname_unicode,
2646                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2647                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2648                                                   u"sub"]))
2649         return d
2650
2651     def test_POST_FILEURL_check(self):
2652         bar_url = self.public_url + "/foo/bar.txt"
2653         d = self.POST(bar_url, t="check")
2654         def _check(res):
2655             self.failUnlessIn("Healthy :", res)
2656         d.addCallback(_check)
2657         redir_url = "http://allmydata.org/TARGET"
2658         def _check2(statuscode, target):
2659             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2660             self.failUnlessReallyEqual(target, redir_url)
2661         d.addCallback(lambda res:
2662                       self.shouldRedirect2("test_POST_FILEURL_check",
2663                                            _check2,
2664                                            self.POST, bar_url,
2665                                            t="check",
2666                                            when_done=redir_url))
2667         d.addCallback(lambda res:
2668                       self.POST(bar_url, t="check", return_to=redir_url))
2669         def _check3(res):
2670             self.failUnlessIn("Healthy :", res)
2671             self.failUnlessIn("Return to file", res)
2672             self.failUnlessIn(redir_url, res)
2673         d.addCallback(_check3)
2674
2675         d.addCallback(lambda res:
2676                       self.POST(bar_url, t="check", output="JSON"))
2677         def _check_json(res):
2678             data = simplejson.loads(res)
2679             self.failUnlessIn("storage-index", data)
2680             self.failUnless(data["results"]["healthy"])
2681         d.addCallback(_check_json)
2682
2683         return d
2684
2685     def test_POST_FILEURL_check_and_repair(self):
2686         bar_url = self.public_url + "/foo/bar.txt"
2687         d = self.POST(bar_url, t="check", repair="true")
2688         def _check(res):
2689             self.failUnlessIn("Healthy :", res)
2690         d.addCallback(_check)
2691         redir_url = "http://allmydata.org/TARGET"
2692         def _check2(statuscode, target):
2693             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2694             self.failUnlessReallyEqual(target, redir_url)
2695         d.addCallback(lambda res:
2696                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2697                                            _check2,
2698                                            self.POST, bar_url,
2699                                            t="check", repair="true",
2700                                            when_done=redir_url))
2701         d.addCallback(lambda res:
2702                       self.POST(bar_url, t="check", return_to=redir_url))
2703         def _check3(res):
2704             self.failUnlessIn("Healthy :", res)
2705             self.failUnlessIn("Return to file", res)
2706             self.failUnlessIn(redir_url, res)
2707         d.addCallback(_check3)
2708         return d
2709
2710     def test_POST_DIRURL_check(self):
2711         foo_url = self.public_url + "/foo/"
2712         d = self.POST(foo_url, t="check")
2713         def _check(res):
2714             self.failUnlessIn("Healthy :", res)
2715         d.addCallback(_check)
2716         redir_url = "http://allmydata.org/TARGET"
2717         def _check2(statuscode, target):
2718             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2719             self.failUnlessReallyEqual(target, redir_url)
2720         d.addCallback(lambda res:
2721                       self.shouldRedirect2("test_POST_DIRURL_check",
2722                                            _check2,
2723                                            self.POST, foo_url,
2724                                            t="check",
2725                                            when_done=redir_url))
2726         d.addCallback(lambda res:
2727                       self.POST(foo_url, t="check", return_to=redir_url))
2728         def _check3(res):
2729             self.failUnlessIn("Healthy :", res)
2730             self.failUnlessIn("Return to file/directory", res)
2731             self.failUnlessIn(redir_url, res)
2732         d.addCallback(_check3)
2733
2734         d.addCallback(lambda res:
2735                       self.POST(foo_url, t="check", output="JSON"))
2736         def _check_json(res):
2737             data = simplejson.loads(res)
2738             self.failUnlessIn("storage-index", data)
2739             self.failUnless(data["results"]["healthy"])
2740         d.addCallback(_check_json)
2741
2742         return d
2743
2744     def test_POST_DIRURL_check_and_repair(self):
2745         foo_url = self.public_url + "/foo/"
2746         d = self.POST(foo_url, t="check", repair="true")
2747         def _check(res):
2748             self.failUnlessIn("Healthy :", res)
2749         d.addCallback(_check)
2750         redir_url = "http://allmydata.org/TARGET"
2751         def _check2(statuscode, target):
2752             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2753             self.failUnlessReallyEqual(target, redir_url)
2754         d.addCallback(lambda res:
2755                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2756                                            _check2,
2757                                            self.POST, foo_url,
2758                                            t="check", repair="true",
2759                                            when_done=redir_url))
2760         d.addCallback(lambda res:
2761                       self.POST(foo_url, t="check", return_to=redir_url))
2762         def _check3(res):
2763             self.failUnlessIn("Healthy :", res)
2764             self.failUnlessIn("Return to file/directory", res)
2765             self.failUnlessIn(redir_url, res)
2766         d.addCallback(_check3)
2767         return d
2768
2769     def test_POST_FILEURL_mdmf_check(self):
2770         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2771         d = self.POST(quux_url, t="check")
2772         def _check(res):
2773             self.failUnlessIn("Healthy", res)
2774         d.addCallback(_check)
2775         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2776         d.addCallback(lambda ignored:
2777                       self.POST(quux_extension_url, t="check"))
2778         d.addCallback(_check)
2779         return d
2780
2781     def test_POST_FILEURL_mdmf_check_and_repair(self):
2782         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2783         d = self.POST(quux_url, t="check", repair="true")
2784         def _check(res):
2785             self.failUnlessIn("Healthy", res)
2786         d.addCallback(_check)
2787         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2788         d.addCallback(lambda ignored:
2789                       self.POST(quux_extension_url, t="check", repair="true"))
2790         d.addCallback(_check)
2791         return d
2792
2793     def wait_for_operation(self, ignored, ophandle):
2794         url = "/operations/" + ophandle
2795         url += "?t=status&output=JSON"
2796         d = self.GET(url)
2797         def _got(res):
2798             data = simplejson.loads(res)
2799             if not data["finished"]:
2800                 d = self.stall(delay=1.0)
2801                 d.addCallback(self.wait_for_operation, ophandle)
2802                 return d
2803             return data
2804         d.addCallback(_got)
2805         return d
2806
2807     def get_operation_results(self, ignored, ophandle, output=None):
2808         url = "/operations/" + ophandle
2809         url += "?t=status"
2810         if output:
2811             url += "&output=" + output
2812         d = self.GET(url)
2813         def _got(res):
2814             if output and output.lower() == "json":
2815                 return simplejson.loads(res)
2816             return res
2817         d.addCallback(_got)
2818         return d
2819
2820     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2821         d = self.shouldFail2(error.Error,
2822                              "test_POST_DIRURL_deepcheck_no_ophandle",
2823                              "400 Bad Request",
2824                              "slow operation requires ophandle=",
2825                              self.POST, self.public_url, t="start-deep-check")
2826         return d
2827
2828     def test_POST_DIRURL_deepcheck(self):
2829         def _check_redirect(statuscode, target):
2830             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2831             self.failUnless(target.endswith("/operations/123"))
2832         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2833                                  self.POST, self.public_url,
2834                                  t="start-deep-check", ophandle="123")
2835         d.addCallback(self.wait_for_operation, "123")
2836         def _check_json(data):
2837             self.failUnlessReallyEqual(data["finished"], True)
2838             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2839             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2840         d.addCallback(_check_json)
2841         d.addCallback(self.get_operation_results, "123", "html")
2842         def _check_html(res):
2843             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2844             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2845             self.failUnlessIn(FAVICON_MARKUP, res)
2846         d.addCallback(_check_html)
2847
2848         d.addCallback(lambda res:
2849                       self.GET("/operations/123/"))
2850         d.addCallback(_check_html) # should be the same as without the slash
2851
2852         d.addCallback(lambda res:
2853                       self.shouldFail2(error.Error, "one", "404 Not Found",
2854                                        "No detailed results for SI bogus",
2855                                        self.GET, "/operations/123/bogus"))
2856
2857         foo_si = self._foo_node.get_storage_index()
2858         foo_si_s = base32.b2a(foo_si)
2859         d.addCallback(lambda res:
2860                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2861         def _check_foo_json(res):
2862             data = simplejson.loads(res)
2863             self.failUnlessEqual(data["storage-index"], foo_si_s)
2864             self.failUnless(data["results"]["healthy"])
2865         d.addCallback(_check_foo_json)
2866         return d
2867
2868     def test_POST_DIRURL_deepcheck_and_repair(self):
2869         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2870                       ophandle="124", output="json", followRedirect=True)
2871         d.addCallback(self.wait_for_operation, "124")
2872         def _check_json(data):
2873             self.failUnlessReallyEqual(data["finished"], True)
2874             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2875             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2876             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2877             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2878             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2879             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2880             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2881             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2882             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2883             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2884         d.addCallback(_check_json)
2885         d.addCallback(self.get_operation_results, "124", "html")
2886         def _check_html(res):
2887             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2888
2889             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2890             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2891             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2892
2893             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2894             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2895             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2896
2897             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2898             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2899             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2900
2901             self.failUnlessIn(FAVICON_MARKUP, res)
2902         d.addCallback(_check_html)
2903         return d
2904
2905     def test_POST_FILEURL_bad_t(self):
2906         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2907                              "POST to file: bad t=bogus",
2908                              self.POST, self.public_url + "/foo/bar.txt",
2909                              t="bogus")
2910         return d
2911
2912     def test_POST_mkdir(self): # return value?
2913         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2914         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2915         d.addCallback(self.failUnlessNodeKeysAre, [])
2916         return d
2917
2918     def test_POST_mkdir_mdmf(self):
2919         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2920         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2921         d.addCallback(lambda node:
2922             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2923         return d
2924
2925     def test_POST_mkdir_sdmf(self):
2926         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2928         d.addCallback(lambda node:
2929             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2930         return d
2931
2932     def test_POST_mkdir_bad_format(self):
2933         return self.shouldHTTPError("POST_mkdir_bad_format",
2934                                     400, "Bad Request", "Unknown format: foo",
2935                                     self.POST, self.public_url +
2936                                     "/foo?t=mkdir&name=newdir&format=foo")
2937
2938     def test_POST_mkdir_initial_children(self):
2939         (newkids, caps) = self._create_initial_children()
2940         d = self.POST2(self.public_url +
2941                        "/foo?t=mkdir-with-children&name=newdir",
2942                        simplejson.dumps(newkids))
2943         d.addCallback(lambda res:
2944                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2945         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2946         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2947         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2949         return d
2950
2951     def test_POST_mkdir_initial_children_mdmf(self):
2952         (newkids, caps) = self._create_initial_children()
2953         d = self.POST2(self.public_url +
2954                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2955                        simplejson.dumps(newkids))
2956         d.addCallback(lambda res:
2957                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2958         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2959         d.addCallback(lambda node:
2960             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2961         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2962         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2963                        caps['filecap1'])
2964         return d
2965
2966     # XXX: Duplication.
2967     def test_POST_mkdir_initial_children_sdmf(self):
2968         (newkids, caps) = self._create_initial_children()
2969         d = self.POST2(self.public_url +
2970                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2971                        simplejson.dumps(newkids))
2972         d.addCallback(lambda res:
2973                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2974         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2975         d.addCallback(lambda node:
2976             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2977         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2978         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2979                        caps['filecap1'])
2980         return d
2981
2982     def test_POST_mkdir_initial_children_bad_format(self):
2983         (newkids, caps) = self._create_initial_children()
2984         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2985                                     400, "Bad Request", "Unknown format: foo",
2986                                     self.POST, self.public_url + \
2987                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2988                                     simplejson.dumps(newkids))
2989
2990     def test_POST_mkdir_immutable(self):
2991         (newkids, caps) = self._create_immutable_children()
2992         d = self.POST2(self.public_url +
2993                        "/foo?t=mkdir-immutable&name=newdir",
2994                        simplejson.dumps(newkids))
2995         d.addCallback(lambda res:
2996                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3007         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3008         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3009         return d
3010
3011     def test_POST_mkdir_immutable_bad(self):
3012         (newkids, caps) = self._create_initial_children()
3013         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3014                              "400 Bad Request",
3015                              "needed to be immutable but was not",
3016                              self.POST2,
3017                              self.public_url +
3018                              "/foo?t=mkdir-immutable&name=newdir",
3019                              simplejson.dumps(newkids))
3020         return d
3021
3022     def test_POST_mkdir_2(self):
3023         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3024         d.addCallback(lambda res:
3025                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3026         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3027         d.addCallback(self.failUnlessNodeKeysAre, [])
3028         return d
3029
3030     def test_POST_mkdirs_2(self):
3031         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3032         d.addCallback(lambda res:
3033                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3034         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3035         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3036         d.addCallback(self.failUnlessNodeKeysAre, [])
3037         return d
3038
3039     def test_POST_mkdir_no_parentdir_noredirect(self):
3040         d = self.POST("/uri?t=mkdir")
3041         def _after_mkdir(res):
3042             uri.DirectoryURI.init_from_string(res)
3043         d.addCallback(_after_mkdir)
3044         return d
3045
3046     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3047         d = self.POST("/uri?t=mkdir&format=mdmf")
3048         def _after_mkdir(res):
3049             u = uri.from_string(res)
3050             # Check that this is an MDMF writecap
3051             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3052         d.addCallback(_after_mkdir)
3053         return d
3054
3055     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3056         d = self.POST("/uri?t=mkdir&format=sdmf")
3057         def _after_mkdir(res):
3058             u = uri.from_string(res)
3059             self.failUnlessIsInstance(u, uri.DirectoryURI)
3060         d.addCallback(_after_mkdir)
3061         return d
3062
3063     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3064         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3065                                     400, "Bad Request", "Unknown format: foo",
3066                                     self.POST, self.public_url +
3067                                     "/uri?t=mkdir&format=foo")
3068
3069     def test_POST_mkdir_no_parentdir_noredirect2(self):
3070         # make sure form-based arguments (as on the welcome page) still work
3071         d = self.POST("/uri", t="mkdir")
3072         def _after_mkdir(res):
3073             uri.DirectoryURI.init_from_string(res)
3074         d.addCallback(_after_mkdir)
3075         d.addErrback(self.explain_web_error)
3076         return d
3077
3078     def test_POST_mkdir_no_parentdir_redirect(self):
3079         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3080         d.addBoth(self.shouldRedirect, None, statuscode='303')
3081         def _check_target(target):
3082             target = urllib.unquote(target)
3083             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3084         d.addCallback(_check_target)
3085         return d
3086
3087     def test_POST_mkdir_no_parentdir_redirect2(self):
3088         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3089         d.addBoth(self.shouldRedirect, None, statuscode='303')
3090         def _check_target(target):
3091             target = urllib.unquote(target)
3092             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3093         d.addCallback(_check_target)
3094         d.addErrback(self.explain_web_error)
3095         return d
3096
3097     def _make_readonly(self, u):
3098         ro_uri = uri.from_string(u).get_readonly()
3099         if ro_uri is None:
3100             return None
3101         return ro_uri.to_string()
3102
3103     def _create_initial_children(self):
3104         contents, n, filecap1 = self.makefile(12)
3105         md1 = {"metakey1": "metavalue1"}
3106         filecap2 = make_mutable_file_uri()
3107         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3108         filecap3 = node3.get_readonly_uri()
3109         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3110         dircap = DirectoryNode(node4, None, None).get_uri()
3111         mdmfcap = make_mutable_file_uri(mdmf=True)
3112         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3113         emptydircap = "URI:DIR2-LIT:"
3114         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3115                                                       "ro_uri": self._make_readonly(filecap1),
3116                                                       "metadata": md1, }],
3117                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3118                                                       "ro_uri": self._make_readonly(filecap2)}],
3119                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3120                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3121                                                       "ro_uri": unknown_rocap}],
3122                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3123                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3124                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3125                                                       "ro_uri": self._make_readonly(dircap)}],
3126                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3127                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3128                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3129                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3130                    }
3131         return newkids, {'filecap1': filecap1,
3132                          'filecap2': filecap2,
3133                          'filecap3': filecap3,
3134                          'unknown_rwcap': unknown_rwcap,
3135                          'unknown_rocap': unknown_rocap,
3136                          'unknown_immcap': unknown_immcap,
3137                          'dircap': dircap,
3138                          'litdircap': litdircap,
3139                          'emptydircap': emptydircap,
3140                          'mdmfcap': mdmfcap}
3141
3142     def _create_immutable_children(self):
3143         contents, n, filecap1 = self.makefile(12)
3144         md1 = {"metakey1": "metavalue1"}
3145         tnode = create_chk_filenode("immutable directory contents\n"*10,
3146                                     self.get_all_contents())
3147         dnode = DirectoryNode(tnode, None, None)
3148         assert not dnode.is_mutable()
3149         immdircap = dnode.get_uri()
3150         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3151         emptydircap = "URI:DIR2-LIT:"
3152         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3153                                                       "metadata": md1, }],
3154                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3155                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3156                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3157                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3158                    }
3159         return newkids, {'filecap1': filecap1,
3160                          'unknown_immcap': unknown_immcap,
3161                          'immdircap': immdircap,
3162                          'litdircap': litdircap,
3163                          'emptydircap': emptydircap}
3164
3165     def test_POST_mkdir_no_parentdir_initial_children(self):
3166         (newkids, caps) = self._create_initial_children()
3167         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3168         def _after_mkdir(res):
3169             self.failUnless(res.startswith("URI:DIR"), res)
3170             n = self.s.create_node_from_uri(res)
3171             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3172             d2.addCallback(lambda ign:
3173                            self.failUnlessROChildURIIs(n, u"child-imm",
3174                                                        caps['filecap1']))
3175             d2.addCallback(lambda ign:
3176                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3177                                                        caps['filecap2']))
3178             d2.addCallback(lambda ign:
3179                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3180                                                        caps['filecap3']))
3181             d2.addCallback(lambda ign:
3182                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3183                                                        caps['unknown_rwcap']))
3184             d2.addCallback(lambda ign:
3185                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3186                                                        caps['unknown_rocap']))
3187             d2.addCallback(lambda ign:
3188                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3189                                                        caps['unknown_immcap']))
3190             d2.addCallback(lambda ign:
3191                            self.failUnlessRWChildURIIs(n, u"dirchild",
3192                                                        caps['dircap']))
3193             return d2
3194         d.addCallback(_after_mkdir)
3195         return d
3196
3197     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3198         # the regular /uri?t=mkdir operation is specified to ignore its body.
3199         # Only t=mkdir-with-children pays attention to it.
3200         (newkids, caps) = self._create_initial_children()
3201         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3202                                  400, "Bad Request",
3203                                  "t=mkdir does not accept children=, "
3204                                  "try t=mkdir-with-children instead",
3205                                  self.POST2, "/uri?t=mkdir", # without children
3206                                  simplejson.dumps(newkids))
3207         return d
3208
3209     def test_POST_noparent_bad(self):
3210         d = self.shouldHTTPError("POST_noparent_bad",
3211                                  400, "Bad Request",
3212                                  "/uri accepts only PUT, PUT?t=mkdir, "
3213                                  "POST?t=upload, and POST?t=mkdir",
3214                                  self.POST, "/uri?t=bogus")
3215         return d
3216
3217     def test_POST_mkdir_no_parentdir_immutable(self):
3218         (newkids, caps) = self._create_immutable_children()
3219         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3220         def _after_mkdir(res):
3221             self.failUnless(res.startswith("URI:DIR"), res)
3222             n = self.s.create_node_from_uri(res)
3223             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3224             d2.addCallback(lambda ign:
3225                            self.failUnlessROChildURIIs(n, u"child-imm",
3226                                                           caps['filecap1']))
3227             d2.addCallback(lambda ign:
3228                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3229                                                           caps['unknown_immcap']))
3230             d2.addCallback(lambda ign:
3231                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3232                                                           caps['immdircap']))
3233             d2.addCallback(lambda ign:
3234                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3235                                                           caps['litdircap']))
3236             d2.addCallback(lambda ign:
3237                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3238                                                           caps['emptydircap']))
3239             return d2
3240         d.addCallback(_after_mkdir)
3241         return d
3242
3243     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3244         (newkids, caps) = self._create_initial_children()
3245         d = self.shouldFail2(error.Error,
3246                              "test_POST_mkdir_no_parentdir_immutable_bad",
3247                              "400 Bad Request",
3248                              "needed to be immutable but was not",
3249                              self.POST2,
3250                              "/uri?t=mkdir-immutable",
3251                              simplejson.dumps(newkids))
3252         return d
3253
3254     def test_welcome_page_mkdir_button(self):
3255         # Fetch the welcome page.
3256         d = self.GET("/")
3257         def _after_get_welcome_page(res):
3258             MKDIR_BUTTON_RE = re.compile(
3259                 '<form action="([^"]*)" method="post".*'
3260                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3261                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3262                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3263             html = res.replace('\n', ' ')
3264             mo = MKDIR_BUTTON_RE.search(html)
3265             self.failUnless(mo, html)
3266             formaction = mo.group(1)
3267             formt = mo.group(2)
3268             formaname = mo.group(3)
3269             formavalue = mo.group(4)
3270             return (formaction, formt, formaname, formavalue)
3271         d.addCallback(_after_get_welcome_page)
3272         def _after_parse_form(res):
3273             (formaction, formt, formaname, formavalue) = res
3274             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3275         d.addCallback(_after_parse_form)
3276         d.addBoth(self.shouldRedirect, None, statuscode='303')
3277         return d
3278
3279     def test_POST_mkdir_replace(self): # return value?
3280         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3281         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3282         d.addCallback(self.failUnlessNodeKeysAre, [])
3283         return d
3284
3285     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3286         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3287         d.addBoth(self.shouldFail, error.Error,
3288                   "POST_mkdir_no_replace_queryarg",
3289                   "409 Conflict",
3290                   "There was already a child by that name, and you asked me "
3291                   "to not replace it")
3292         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3293         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3294         return d
3295
3296     def test_POST_mkdir_no_replace_field(self): # return value?
3297         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3298                       replace="false")
3299         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3300                   "409 Conflict",
3301                   "There was already a child by that name, and you asked me "
3302                   "to not replace it")
3303         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3304         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3305         return d
3306
3307     def test_POST_mkdir_whendone_field(self):
3308         d = self.POST(self.public_url + "/foo",
3309                       t="mkdir", name="newdir", when_done="/THERE")
3310         d.addBoth(self.shouldRedirect, "/THERE")
3311         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3312         d.addCallback(self.failUnlessNodeKeysAre, [])
3313         return d
3314
3315     def test_POST_mkdir_whendone_queryarg(self):
3316         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3317                       t="mkdir", name="newdir")
3318         d.addBoth(self.shouldRedirect, "/THERE")
3319         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3320         d.addCallback(self.failUnlessNodeKeysAre, [])
3321         return d
3322
3323     def test_POST_bad_t(self):
3324         d = self.shouldFail2(error.Error, "POST_bad_t",
3325                              "400 Bad Request",
3326                              "POST to a directory with bad t=BOGUS",
3327                              self.POST, self.public_url + "/foo", t="BOGUS")
3328         return d
3329
3330     def test_POST_set_children(self, command_name="set_children"):
3331         contents9, n9, newuri9 = self.makefile(9)
3332         contents10, n10, newuri10 = self.makefile(10)
3333         contents11, n11, newuri11 = self.makefile(11)
3334
3335         reqbody = """{
3336                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3337                                                 "size": 0,
3338                                                 "metadata": {
3339                                                   "ctime": 1002777696.7564139,
3340                                                   "mtime": 1002777696.7564139
3341                                                  }
3342                                                } ],
3343                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3344                                                 "size": 1,
3345                                                 "metadata": {
3346                                                   "ctime": 1002777696.7564139,
3347                                                   "mtime": 1002777696.7564139
3348                                                  }
3349                                                } ],
3350                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3351                                                 "size": 2,
3352                                                 "metadata": {
3353                                                   "ctime": 1002777696.7564139,
3354                                                   "mtime": 1002777696.7564139
3355                                                  }
3356                                                } ]
3357                     }""" % (newuri9, newuri10, newuri11)
3358
3359         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3360
3361         d = client.getPage(url, method="POST", postdata=reqbody)
3362         def _then(res):
3363             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3364             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3365             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3366
3367         d.addCallback(_then)
3368         d.addErrback(self.dump_error)
3369         return d
3370
3371     def test_POST_set_children_with_hyphen(self):
3372         return self.test_POST_set_children(command_name="set-children")
3373
3374     def test_POST_link_uri(self):
3375         contents, n, newuri = self.makefile(8)
3376         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3377         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3378         d.addCallback(lambda res:
3379                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3380                                                       contents))
3381         return d
3382
3383     def test_POST_link_uri_replace(self):
3384         contents, n, newuri = self.makefile(8)
3385         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3386         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3387         d.addCallback(lambda res:
3388                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3389                                                       contents))
3390         return d
3391
3392     def test_POST_link_uri_unknown_bad(self):
3393         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3394         d.addBoth(self.shouldFail, error.Error,
3395                   "POST_link_uri_unknown_bad",
3396                   "400 Bad Request",
3397                   "unknown cap in a write slot")
3398         return d
3399
3400     def test_POST_link_uri_unknown_ro_good(self):
3401         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3402         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3403         return d
3404
3405     def test_POST_link_uri_unknown_imm_good(self):
3406         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3407         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3408         return d
3409
3410     def test_POST_link_uri_no_replace_queryarg(self):
3411         contents, n, newuri = self.makefile(8)
3412         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3413                       name="bar.txt", uri=newuri)
3414         d.addBoth(self.shouldFail, error.Error,
3415                   "POST_link_uri_no_replace_queryarg",
3416                   "409 Conflict",
3417                   "There was already a child by that name, and you asked me "
3418                   "to not replace it")
3419         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3420         d.addCallback(self.failUnlessIsBarDotTxt)
3421         return d
3422
3423     def test_POST_link_uri_no_replace_field(self):
3424         contents, n, newuri = self.makefile(8)
3425         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3426                       name="bar.txt", uri=newuri)
3427         d.addBoth(self.shouldFail, error.Error,
3428                   "POST_link_uri_no_replace_field",
3429                   "409 Conflict",
3430                   "There was already a child by that name, and you asked me "
3431                   "to not replace it")
3432         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3433         d.addCallback(self.failUnlessIsBarDotTxt)
3434         return d
3435
3436     def test_POST_delete(self, command_name='delete'):
3437         d = self._foo_node.list()
3438         def _check_before(children):
3439             self.failUnlessIn(u"bar.txt", children)
3440         d.addCallback(_check_before)
3441         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3442         d.addCallback(lambda res: self._foo_node.list())
3443         def _check_after(children):
3444             self.failIfIn(u"bar.txt", children)
3445         d.addCallback(_check_after)
3446         return d
3447
3448     def test_POST_unlink(self):
3449         return self.test_POST_delete(command_name='unlink')
3450
3451     def test_POST_rename_file(self):
3452         d = self.POST(self.public_url + "/foo", t="rename",
3453                       from_name="bar.txt", to_name='wibble.txt')
3454         d.addCallback(lambda res:
3455                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3456         d.addCallback(lambda res:
3457                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3458         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3459         d.addCallback(self.failUnlessIsBarDotTxt)
3460         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3461         d.addCallback(self.failUnlessIsBarJSON)
3462         return d
3463
3464     def test_POST_rename_file_redundant(self):
3465         d = self.POST(self.public_url + "/foo", t="rename",
3466                       from_name="bar.txt", to_name='bar.txt')
3467         d.addCallback(lambda res:
3468                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3470         d.addCallback(self.failUnlessIsBarDotTxt)
3471         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3472         d.addCallback(self.failUnlessIsBarJSON)
3473         return d
3474
3475     def test_POST_rename_file_replace(self):
3476         # rename a file and replace a directory with it
3477         d = self.POST(self.public_url + "/foo", t="rename",
3478                       from_name="bar.txt", to_name='empty')
3479         d.addCallback(lambda res:
3480                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3481         d.addCallback(lambda res:
3482                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3483         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3484         d.addCallback(self.failUnlessIsBarDotTxt)
3485         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3486         d.addCallback(self.failUnlessIsBarJSON)
3487         return d
3488
3489     def test_POST_rename_file_no_replace_queryarg(self):
3490         # rename a file and replace a directory with it
3491         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3492                       from_name="bar.txt", to_name='empty')
3493         d.addBoth(self.shouldFail, error.Error,
3494                   "POST_rename_file_no_replace_queryarg",
3495                   "409 Conflict",
3496                   "There was already a child by that name, and you asked me "
3497                   "to not replace it")
3498         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3499         d.addCallback(self.failUnlessIsEmptyJSON)
3500         return d
3501
3502     def test_POST_rename_file_no_replace_field(self):
3503         # rename a file and replace a directory with it
3504         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3505                       from_name="bar.txt", to_name='empty')
3506         d.addBoth(self.shouldFail, error.Error,
3507                   "POST_rename_file_no_replace_field",
3508                   "409 Conflict",
3509                   "There was already a child by that name, and you asked me "
3510                   "to not replace it")
3511         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3512         d.addCallback(self.failUnlessIsEmptyJSON)
3513         return d
3514
3515     def test_POST_rename_file_no_replace_same_link(self):
3516         d = self.POST(self.public_url + "/foo", t="rename",
3517                       replace="false", from_name="bar.txt", to_name="bar.txt")
3518         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3520         d.addCallback(self.failUnlessIsBarDotTxt)
3521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3522         d.addCallback(self.failUnlessIsBarJSON)
3523         return d
3524
3525     def test_POST_rename_file_replace_only_files(self):
3526         d = self.POST(self.public_url + "/foo", t="rename",
3527                       replace="only-files", from_name="bar.txt",
3528                       to_name="baz.txt")
3529         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3530         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3531         d.addCallback(self.failUnlessIsBarDotTxt)
3532         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3533         d.addCallback(self.failUnlessIsBarJSON)
3534         return d
3535
3536     def test_POST_rename_file_replace_only_files_conflict(self):
3537         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3538                              "409 Conflict",
3539                              "There was already a child by that name, and you asked me to not replace it.",
3540                              self.POST, self.public_url + "/foo", t="relink",
3541                              replace="only-files", from_name="bar.txt",
3542                              to_name="empty")
3543         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3544         d.addCallback(self.failUnlessIsBarDotTxt)
3545         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3546         d.addCallback(self.failUnlessIsBarJSON)
3547         return d
3548
3549     def failUnlessIsEmptyJSON(self, res):
3550         data = simplejson.loads(res)
3551         self.failUnlessEqual(data[0], "dirnode", data)
3552         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3553
3554     def test_POST_rename_file_to_slash_fail(self):
3555         d = self.POST(self.public_url + "/foo", t="rename",
3556                       from_name="bar.txt", to_name='kirk/spock.txt')
3557         d.addBoth(self.shouldFail, error.Error,
3558                   "test_POST_rename_file_to_slash_fail",
3559                   "400 Bad Request",
3560                   "to_name= may not contain a slash",
3561                   )
3562         d.addCallback(lambda res:
3563                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3564         return d
3565
3566     def test_POST_rename_file_from_slash_fail(self):
3567         d = self.POST(self.public_url + "/foo", t="rename",
3568                       from_name="sub/bar.txt", to_name='spock.txt')
3569         d.addBoth(self.shouldFail, error.Error,
3570                   "test_POST_rename_from_file_slash_fail",
3571                   "400 Bad Request",
3572                   "from_name= may not contain a slash",
3573                   )
3574         d.addCallback(lambda res:
3575                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3576         return d
3577
3578     def test_POST_rename_dir(self):
3579         d = self.POST(self.public_url, t="rename",
3580                       from_name="foo", to_name='plunk')
3581         d.addCallback(lambda res:
3582                       self.failIfNodeHasChild(self.public_root, u"foo"))
3583         d.addCallback(lambda res:
3584                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3585         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3586         d.addCallback(self.failUnlessIsFooJSON)
3587         return d
3588
3589     def test_POST_relink_file(self):
3590         d = self.POST(self.public_url + "/foo", t="relink",
3591                       from_name="bar.txt",
3592                       to_dir=self.public_root.get_uri() + "/foo/sub")
3593         d.addCallback(lambda res:
3594                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3595         d.addCallback(lambda res:
3596                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3597         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3598         d.addCallback(self.failUnlessIsBarDotTxt)
3599         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3600         d.addCallback(self.failUnlessIsBarJSON)
3601         return d
3602
3603     def test_POST_relink_file_new_name(self):
3604         d = self.POST(self.public_url + "/foo", t="relink",
3605                       from_name="bar.txt",
3606                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3607         d.addCallback(lambda res:
3608                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3609         d.addCallback(lambda res:
3610                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3611         d.addCallback(lambda res:
3612                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3613         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3614         d.addCallback(self.failUnlessIsBarDotTxt)
3615         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3616         d.addCallback(self.failUnlessIsBarJSON)
3617         return d
3618
3619     def test_POST_relink_file_replace(self):
3620         d = self.POST(self.public_url + "/foo", t="relink",
3621                       from_name="bar.txt",
3622                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3623         d.addCallback(lambda res:
3624                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3625         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3626         d.addCallback(self.failUnlessIsBarDotTxt)
3627         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3628         d.addCallback(self.failUnlessIsBarJSON)
3629         return d
3630
3631     def test_POST_relink_file_no_replace(self):
3632         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3633                              "409 Conflict",
3634                              "There was already a child by that name, and you asked me to not replace it",
3635                              self.POST, self.public_url + "/foo", t="relink",
3636                              replace="false", from_name="bar.txt",
3637                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3638         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3639         d.addCallback(self.failUnlessIsBarDotTxt)
3640         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3641         d.addCallback(self.failUnlessIsBarJSON)
3642         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3643         d.addCallback(self.failUnlessIsSubBazDotTxt)
3644         return d
3645
3646     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3647         d = self.POST(self.public_url + "/foo", t="relink",
3648                       replace="false", from_name="bar.txt",
3649                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3650         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3651         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3652         d.addCallback(self.failUnlessIsBarDotTxt)
3653         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3654         d.addCallback(self.failUnlessIsBarJSON)
3655         return d
3656
3657     def test_POST_relink_file_replace_only_files(self):
3658         d = self.POST(self.public_url + "/foo", t="relink",
3659                       replace="only-files", from_name="bar.txt",
3660                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3661         d.addCallback(lambda res:
3662                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3663         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3664         d.addCallback(self.failUnlessIsBarDotTxt)
3665         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3666         d.addCallback(self.failUnlessIsBarJSON)
3667         return d
3668
3669     def test_POST_relink_file_replace_only_files_conflict(self):
3670         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3671                              "409 Conflict",
3672                              "There was already a child by that name, and you asked me to not replace it.",
3673                              self.POST, self.public_url + "/foo", t="relink",
3674                              replace="only-files", from_name="bar.txt",
3675                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3676         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3677         d.addCallback(self.failUnlessIsBarDotTxt)
3678         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3679         d.addCallback(self.failUnlessIsBarJSON)
3680         return d
3681
3682     def test_POST_relink_file_to_slash_fail(self):
3683         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3684                              "400 Bad Request",
3685                              "to_name= may not contain a slash",
3686                              self.POST, self.public_url + "/foo", t="relink",
3687                              from_name="bar.txt",
3688                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3689         d.addCallback(lambda res:
3690                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3691         d.addCallback(lambda res:
3692                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3693         d.addCallback(lambda ign:
3694                       self.shouldFail2(error.Error,
3695                                        "test_POST_rename_file_slash_fail2",
3696                                        "400 Bad Request",
3697                                        "from_name= may not contain a slash",
3698                                        self.POST, self.public_url + "/foo",
3699                                        t="relink",
3700                                        from_name="nope/bar.txt",
3701                                        to_name="fail.txt",
3702                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3703         return d
3704
3705     def test_POST_relink_file_explicitly_same_link(self):
3706         d = self.POST(self.public_url + "/foo", t="relink",
3707                       from_name="bar.txt",
3708                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3709         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3710         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3711         d.addCallback(self.failUnlessIsBarDotTxt)
3712         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3713         d.addCallback(self.failUnlessIsBarJSON)
3714         return d
3715
3716     def test_POST_relink_file_implicitly_same_link(self):
3717         d = self.POST(self.public_url + "/foo", t="relink",
3718                       from_name="bar.txt")
3719         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3720         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3721         d.addCallback(self.failUnlessIsBarDotTxt)
3722         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3723         d.addCallback(self.failUnlessIsBarJSON)
3724         return d
3725
3726     def test_POST_relink_file_same_dir(self):
3727         d = self.POST(self.public_url + "/foo", t="relink",
3728                       from_name="bar.txt",
3729                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3730         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3731         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3732         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3733         d.addCallback(self.failUnlessIsBarDotTxt)
3734         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3735         d.addCallback(self.failUnlessIsBarJSON)
3736         return d
3737
3738     def test_POST_relink_file_bad_replace(self):
3739         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3740                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3741                              self.POST,
3742                              self.public_url + "/foo", t="relink",
3743                              replace="boogabooga", from_name="bar.txt",
3744                              to_dir=self.public_root.get_uri() + "/foo/sub")
3745         return d
3746
3747     def test_POST_relink_file_multi_level(self):
3748         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3749         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3750                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3751         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3752         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3753         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3754         d.addCallback(self.failUnlessIsBarDotTxt)
3755         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3756         d.addCallback(self.failUnlessIsBarJSON)
3757         return d
3758
3759     def test_POST_relink_file_to_uri(self):
3760         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3761                       from_name="bar.txt", to_dir=self._sub_uri)
3762         d.addCallback(lambda res:
3763                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3764         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3765         d.addCallback(self.failUnlessIsBarDotTxt)
3766         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3767         d.addCallback(self.failUnlessIsBarJSON)
3768         return d
3769
3770     def test_POST_relink_file_to_nonexistent_dir(self):
3771         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3772                             "404 Not Found", "No such child: nopechucktesta",
3773                             self.POST, self.public_url + "/foo", t="relink",
3774                             from_name="bar.txt",
3775                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3776         return d
3777
3778     def test_POST_relink_file_into_file(self):
3779         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3780                              "400 Bad Request", "to_dir is not a directory",
3781                              self.POST, self.public_url + "/foo", t="relink",
3782                              from_name="bar.txt",
3783                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3784         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3785         d.addCallback(self.failUnlessIsBazDotTxt)
3786         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3787         d.addCallback(self.failUnlessIsBarDotTxt)
3788         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3789         d.addCallback(self.failUnlessIsBarJSON)
3790         return d
3791
3792     def test_POST_relink_file_to_bad_uri(self):
3793         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3794                               "400 Bad Request", "to_dir is not a directory",
3795                               self.POST, self.public_url + "/foo", t="relink",
3796                               from_name="bar.txt",
3797                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3798         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3799         d.addCallback(self.failUnlessIsBarDotTxt)
3800         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3801         d.addCallback(self.failUnlessIsBarJSON)
3802         return d
3803
3804     def test_POST_relink_dir(self):
3805         d = self.POST(self.public_url + "/foo", t="relink",
3806                       from_name="bar.txt",
3807                       to_dir=self.public_root.get_uri() + "/foo/empty")
3808         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3809                       t="relink", from_name="empty",
3810                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3811         d.addCallback(lambda res:
3812                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3813         d.addCallback(lambda res:
3814                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3815         d.addCallback(lambda res:
3816                       self._sub_node.get_child_at_path(u"empty"))
3817         d.addCallback(lambda node:
3818                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3819         d.addCallback(lambda res:
3820                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3821         d.addCallback(self.failUnlessIsBarDotTxt)
3822         return d
3823
3824     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3825         """ If target is not None then the redirection has to go to target.  If
3826         statuscode is not None then the redirection has to be accomplished with
3827         that HTTP status code."""
3828         if not isinstance(res, failure.Failure):
3829             to_where = (target is None) and "somewhere" or ("to " + target)
3830             self.fail("%s: we were expecting to get redirected %s, not get an"
3831                       " actual page: %s" % (which, to_where, res))
3832         res.trap(error.PageRedirect)
3833         if statuscode is not None:
3834             self.failUnlessReallyEqual(res.value.status, statuscode,
3835                                        "%s: not a redirect" % which)
3836         if target is not None:
3837             # the PageRedirect does not seem to capture the uri= query arg
3838             # properly, so we can't check for it.
3839             realtarget = self.webish_url + target
3840             self.failUnlessReallyEqual(res.value.location, realtarget,
3841                                        "%s: wrong target" % which)
3842         return res.value.location
3843
3844     def test_GET_URI_form(self):
3845         base = "/uri?uri=%s" % self._bar_txt_uri
3846         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3847         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3848         d = self.GET(base)
3849         d.addBoth(self.shouldRedirect, targetbase)
3850         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3851         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3852         d.addCallback(lambda res: self.GET(base+"&t=json"))
3853         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3854         d.addCallback(self.log, "about to get file by uri")
3855         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3856         d.addCallback(self.failUnlessIsBarDotTxt)
3857         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3858         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3859                                            followRedirect=True))
3860         d.addCallback(self.failUnlessIsFooJSON)
3861         d.addCallback(self.log, "got dir by uri")
3862
3863         return d
3864
3865     def test_GET_URI_form_bad(self):
3866         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3867                              "400 Bad Request", "GET /uri requires uri=",
3868                              self.GET, "/uri")
3869         return d
3870
3871     def test_GET_rename_form(self):
3872         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3873                      followRedirect=True)
3874         def _check(res):
3875             self.failUnlessIn('name="when_done" value="."', res)
3876             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3877             self.failUnlessIn(FAVICON_MARKUP, res)
3878         d.addCallback(_check)
3879         return d
3880
3881     def log(self, res, msg):
3882         #print "MSG: %s  RES: %s" % (msg, res)
3883         log.msg(msg)
3884         return res
3885
3886     def test_GET_URI_URL(self):
3887         base = "/uri/%s" % self._bar_txt_uri
3888         d = self.GET(base)
3889         d.addCallback(self.failUnlessIsBarDotTxt)
3890         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3891         d.addCallback(self.failUnlessIsBarDotTxt)
3892         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3893         d.addCallback(self.failUnlessIsBarDotTxt)
3894         return d
3895
3896     def test_GET_URI_URL_dir(self):
3897         base = "/uri/%s?t=json" % self._foo_uri
3898         d = self.GET(base)
3899         d.addCallback(self.failUnlessIsFooJSON)
3900         return d
3901
3902     def test_GET_URI_URL_missing(self):
3903         base = "/uri/%s" % self._bad_file_uri
3904         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3905                                  http.GONE, None, "NotEnoughSharesError",
3906                                  self.GET, base)
3907         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3908         # here? we must arrange for a download to fail after target.open()
3909         # has been called, and then inspect the response to see that it is
3910         # shorter than we expected.
3911         return d
3912
3913     def test_PUT_DIRURL_uri(self):
3914         d = self.s.create_dirnode()
3915         def _made_dir(dn):
3916             new_uri = dn.get_uri()
3917             # replace /foo with a new (empty) directory
3918             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3919             d.addCallback(lambda res:
3920                           self.failUnlessReallyEqual(res.strip(), new_uri))
3921             d.addCallback(lambda res:
3922                           self.failUnlessRWChildURIIs(self.public_root,
3923                                                       u"foo",
3924                                                       new_uri))
3925             return d
3926         d.addCallback(_made_dir)
3927         return d
3928
3929     def test_PUT_DIRURL_uri_noreplace(self):
3930         d = self.s.create_dirnode()
3931         def _made_dir(dn):
3932             new_uri = dn.get_uri()
3933             # replace /foo with a new (empty) directory, but ask that
3934             # replace=false, so it should fail
3935             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3936                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3937                                  self.PUT,
3938                                  self.public_url + "/foo?t=uri&replace=false",
3939                                  new_uri)
3940             d.addCallback(lambda res:
3941                           self.failUnlessRWChildURIIs(self.public_root,
3942                                                       u"foo",
3943                                                       self._foo_uri))
3944             return d
3945         d.addCallback(_made_dir)
3946         return d
3947
3948     def test_PUT_DIRURL_bad_t(self):
3949         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3950                              "400 Bad Request", "PUT to a directory",
3951                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3952         d.addCallback(lambda res:
3953                       self.failUnlessRWChildURIIs(self.public_root,
3954                                                   u"foo",
3955                                                   self._foo_uri))
3956         return d
3957
3958     def test_PUT_NEWFILEURL_uri(self):
3959         contents, n, new_uri = self.makefile(8)
3960         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3961         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3962         d.addCallback(lambda res:
3963                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3964                                                       contents))
3965         return d
3966
3967     def test_PUT_NEWFILEURL_mdmf(self):
3968         new_contents = self.NEWFILE_CONTENTS * 300000
3969         d = self.PUT(self.public_url + \
3970                      "/foo/mdmf.txt?format=mdmf",
3971                      new_contents)
3972         d.addCallback(lambda ignored:
3973             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3974         def _got_json(json):
3975             data = simplejson.loads(json)
3976             data = data[1]
3977             self.failUnlessIn("format", data)
3978             self.failUnlessEqual(data["format"], "MDMF")
3979             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3980             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3981         d.addCallback(_got_json)
3982         return d
3983
3984     def test_PUT_NEWFILEURL_sdmf(self):
3985         new_contents = self.NEWFILE_CONTENTS * 300000
3986         d = self.PUT(self.public_url + \
3987                      "/foo/sdmf.txt?format=sdmf",
3988                      new_contents)
3989         d.addCallback(lambda ignored:
3990             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3991         def _got_json(json):
3992             data = simplejson.loads(json)
3993             data = data[1]
3994             self.failUnlessIn("format", data)
3995             self.failUnlessEqual(data["format"], "SDMF")
3996         d.addCallback(_got_json)
3997         return d
3998
3999     def test_PUT_NEWFILEURL_bad_format(self):
4000         new_contents = self.NEWFILE_CONTENTS * 300000
4001         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4002                                     400, "Bad Request", "Unknown format: foo",
4003                                     self.PUT, self.public_url + \
4004                                     "/foo/foo.txt?format=foo",
4005                                     new_contents)
4006
4007     def test_PUT_NEWFILEURL_uri_replace(self):
4008         contents, n, new_uri = self.makefile(8)
4009         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4010         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4011         d.addCallback(lambda res:
4012                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4013                                                       contents))
4014         return d
4015
4016     def test_PUT_NEWFILEURL_uri_no_replace(self):
4017         contents, n, new_uri = self.makefile(8)
4018         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4019         d.addBoth(self.shouldFail, error.Error,
4020                   "PUT_NEWFILEURL_uri_no_replace",
4021                   "409 Conflict",
4022                   "There was already a child by that name, and you asked me "
4023                   "to not replace it")
4024         return d
4025
4026     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4027         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4028         d.addBoth(self.shouldFail, error.Error,
4029                   "POST_put_uri_unknown_bad",
4030                   "400 Bad Request",
4031                   "unknown cap in a write slot")
4032         return d
4033
4034     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4035         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4036         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4037                       u"put-future-ro.txt")
4038         return d
4039
4040     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4041         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4042         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4043                       u"put-future-imm.txt")
4044         return d
4045
4046     def test_PUT_NEWFILE_URI(self):
4047         file_contents = "New file contents here\n"
4048         d = self.PUT("/uri", file_contents)
4049         def _check(uri):
4050             assert isinstance(uri, str), uri
4051             self.failUnlessIn(uri, self.get_all_contents())
4052             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4053                                        file_contents)
4054             return self.GET("/uri/%s" % uri)
4055         d.addCallback(_check)
4056         def _check2(res):
4057             self.failUnlessReallyEqual(res, file_contents)
4058         d.addCallback(_check2)
4059         return d
4060
4061     def test_PUT_NEWFILE_URI_not_mutable(self):
4062         file_contents = "New file contents here\n"
4063         d = self.PUT("/uri?mutable=false", file_contents)
4064         def _check(uri):
4065             assert isinstance(uri, str), uri
4066             self.failUnlessIn(uri, self.get_all_contents())
4067             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4068                                        file_contents)
4069             return self.GET("/uri/%s" % uri)
4070         d.addCallback(_check)
4071         def _check2(res):
4072             self.failUnlessReallyEqual(res, file_contents)
4073         d.addCallback(_check2)
4074         return d
4075
4076     def test_PUT_NEWFILE_URI_only_PUT(self):
4077         d = self.PUT("/uri?t=bogus", "")
4078         d.addBoth(self.shouldFail, error.Error,
4079                   "PUT_NEWFILE_URI_only_PUT",
4080                   "400 Bad Request",
4081                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4082         return d
4083
4084     def test_PUT_NEWFILE_URI_mutable(self):
4085         file_contents = "New file contents here\n"
4086         d = self.PUT("/uri?mutable=true", file_contents)
4087         def _check1(filecap):
4088             filecap = filecap.strip()
4089             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4090             self.filecap = filecap
4091             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4092             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4093             n = self.s.create_node_from_uri(filecap)
4094             return n.download_best_version()
4095         d.addCallback(_check1)
4096         def _check2(data):
4097             self.failUnlessReallyEqual(data, file_contents)
4098             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4099         d.addCallback(_check2)
4100         def _check3(res):
4101             self.failUnlessReallyEqual(res, file_contents)
4102         d.addCallback(_check3)
4103         return d
4104
4105     def test_PUT_mkdir(self):
4106         d = self.PUT("/uri?t=mkdir", "")
4107         def _check(uri):
4108             n = self.s.create_node_from_uri(uri.strip())
4109             d2 = self.failUnlessNodeKeysAre(n, [])
4110             d2.addCallback(lambda res:
4111                            self.GET("/uri/%s?t=json" % uri))
4112             return d2
4113         d.addCallback(_check)
4114         d.addCallback(self.failUnlessIsEmptyJSON)
4115         return d
4116
4117     def test_PUT_mkdir_mdmf(self):
4118         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4119         def _got(res):
4120             u = uri.from_string(res)
4121             # Check that this is an MDMF writecap
4122             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4123         d.addCallback(_got)
4124         return d
4125
4126     def test_PUT_mkdir_sdmf(self):
4127         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4128         def _got(res):
4129             u = uri.from_string(res)
4130             self.failUnlessIsInstance(u, uri.DirectoryURI)
4131         d.addCallback(_got)
4132         return d
4133
4134     def test_PUT_mkdir_bad_format(self):
4135         return self.shouldHTTPError("PUT_mkdir_bad_format",
4136                                     400, "Bad Request", "Unknown format: foo",
4137                                     self.PUT, "/uri?t=mkdir&format=foo",
4138                                     "")
4139
4140     def test_POST_check(self):
4141         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4142         def _done(res):
4143             # this returns a string form of the results, which are probably
4144             # None since we're using fake filenodes.
4145             # TODO: verify that the check actually happened, by changing
4146             # FakeCHKFileNode to count how many times .check() has been
4147             # called.
4148             pass
4149         d.addCallback(_done)
4150         return d
4151
4152
4153     def test_PUT_update_at_offset(self):
4154         file_contents = "test file" * 100000 # about 900 KiB
4155         d = self.PUT("/uri?mutable=true", file_contents)
4156         def _then(filecap):
4157             self.filecap = filecap
4158             new_data = file_contents[:100]
4159             new = "replaced and so on"
4160             new_data += new
4161             new_data += file_contents[len(new_data):]
4162             assert len(new_data) == len(file_contents)
4163             self.new_data = new_data
4164         d.addCallback(_then)
4165         d.addCallback(lambda ignored:
4166             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4167                      "replaced and so on"))
4168         def _get_data(filecap):
4169             n = self.s.create_node_from_uri(filecap)
4170             return n.download_best_version()
4171         d.addCallback(_get_data)
4172         d.addCallback(lambda results:
4173             self.failUnlessEqual(results, self.new_data))
4174         # Now try appending things to the file
4175         d.addCallback(lambda ignored:
4176             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4177                      "puppies" * 100))
4178         d.addCallback(_get_data)
4179         d.addCallback(lambda results:
4180             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4181         # and try replacing the beginning of the file
4182         d.addCallback(lambda ignored:
4183             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4184         d.addCallback(_get_data)
4185         d.addCallback(lambda results:
4186             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4187         return d
4188
4189     def test_PUT_update_at_invalid_offset(self):
4190         file_contents = "test file" * 100000 # about 900 KiB
4191         d = self.PUT("/uri?mutable=true", file_contents)
4192         def _then(filecap):
4193             self.filecap = filecap
4194         d.addCallback(_then)
4195         # Negative offsets should cause an error.
4196         d.addCallback(lambda ignored:
4197             self.shouldHTTPError("PUT_update_at_invalid_offset",
4198                                  400, "Bad Request",
4199                                  "Invalid offset",
4200                                  self.PUT,
4201                                  "/uri/%s?offset=-1" % self.filecap,
4202                                  "foo"))
4203         return d
4204
4205     def test_PUT_update_at_offset_immutable(self):
4206         file_contents = "Test file" * 100000
4207         d = self.PUT("/uri", file_contents)
4208         def _then(filecap):
4209             self.filecap = filecap
4210         d.addCallback(_then)
4211         d.addCallback(lambda ignored:
4212             self.shouldHTTPError("PUT_update_at_offset_immutable",
4213                                  400, "Bad Request",
4214                                  "immutable",
4215                                  self.PUT,
4216                                  "/uri/%s?offset=50" % self.filecap,
4217                                  "foo"))
4218         return d
4219
4220
4221     def test_bad_method(self):
4222         url = self.webish_url + self.public_url + "/foo/bar.txt"
4223         d = self.shouldHTTPError("bad_method",
4224                                  501, "Not Implemented",
4225                                  "I don't know how to treat a BOGUS request.",
4226                                  client.getPage, url, method="BOGUS")
4227         return d
4228
4229     def test_short_url(self):
4230         url = self.webish_url + "/uri"
4231         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4232                                  "I don't know how to treat a DELETE request.",
4233                                  client.getPage, url, method="DELETE")
4234         return d
4235
4236     def test_ophandle_bad(self):
4237         url = self.webish_url + "/operations/bogus?t=status"
4238         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4239                                  "unknown/expired handle 'bogus'",
4240                                  client.getPage, url)
4241         return d
4242
4243     def test_ophandle_cancel(self):
4244         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4245                       followRedirect=True)
4246         d.addCallback(lambda ignored:
4247                       self.GET("/operations/128?t=status&output=JSON"))
4248         def _check1(res):
4249             data = simplejson.loads(res)
4250             self.failUnless("finished" in data, res)
4251             monitor = self.ws.root.child_operations.handles["128"][0]
4252             d = self.POST("/operations/128?t=cancel&output=JSON")
4253             def _check2(res):
4254                 data = simplejson.loads(res)
4255                 self.failUnless("finished" in data, res)
4256                 # t=cancel causes the handle to be forgotten
4257                 self.failUnless(monitor.is_cancelled())
4258             d.addCallback(_check2)
4259             return d
4260         d.addCallback(_check1)
4261         d.addCallback(lambda ignored:
4262                       self.shouldHTTPError("ophandle_cancel",
4263                                            404, "404 Not Found",
4264                                            "unknown/expired handle '128'",
4265                                            self.GET,
4266                                            "/operations/128?t=status&output=JSON"))
4267         return d
4268
4269     def test_ophandle_retainfor(self):
4270         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4271                       followRedirect=True)
4272         d.addCallback(lambda ignored:
4273                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4274         def _check1(res):
4275             data = simplejson.loads(res)
4276             self.failUnless("finished" in data, res)
4277         d.addCallback(_check1)
4278         # the retain-for=0 will cause the handle to be expired very soon
4279         d.addCallback(lambda ign:
4280             self.clock.advance(2.0))
4281         d.addCallback(lambda ignored:
4282                       self.shouldHTTPError("ophandle_retainfor",
4283                                            404, "404 Not Found",
4284                                            "unknown/expired handle '129'",
4285                                            self.GET,
4286                                            "/operations/129?t=status&output=JSON"))
4287         return d
4288
4289     def test_ophandle_release_after_complete(self):
4290         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4291                       followRedirect=True)
4292         d.addCallback(self.wait_for_operation, "130")
4293         d.addCallback(lambda ignored:
4294                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4295         # the release-after-complete=true will cause the handle to be expired
4296         d.addCallback(lambda ignored:
4297                       self.shouldHTTPError("ophandle_release_after_complete",
4298                                            404, "404 Not Found",
4299                                            "unknown/expired handle '130'",
4300                                            self.GET,
4301                                            "/operations/130?t=status&output=JSON"))
4302         return d
4303
4304     def test_uncollected_ophandle_expiration(self):
4305         # uncollected ophandles should expire after 4 days
4306         def _make_uncollected_ophandle(ophandle):
4307             d = self.POST(self.public_url +
4308                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4309                           followRedirect=False)
4310             # When we start the operation, the webapi server will want
4311             # to redirect us to the page for the ophandle, so we get
4312             # confirmation that the operation has started. If the
4313             # manifest operation has finished by the time we get there,
4314             # following that redirect (by setting followRedirect=True
4315             # above) has the side effect of collecting the ophandle that
4316             # we've just created, which means that we can't use the
4317             # ophandle to test the uncollected timeout anymore. So,
4318             # instead, catch the 302 here and don't follow it.
4319             d.addBoth(self.should302, "uncollected_ophandle_creation")
4320             return d
4321         # Create an ophandle, don't collect it, then advance the clock by
4322         # 4 days - 1 second and make sure that the ophandle is still there.
4323         d = _make_uncollected_ophandle(131)
4324         d.addCallback(lambda ign:
4325             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4326         d.addCallback(lambda ign:
4327             self.GET("/operations/131?t=status&output=JSON"))
4328         def _check1(res):
4329             data = simplejson.loads(res)
4330             self.failUnless("finished" in data, res)
4331         d.addCallback(_check1)
4332         # Create an ophandle, don't collect it, then try to collect it
4333         # after 4 days. It should be gone.
4334         d.addCallback(lambda ign:
4335             _make_uncollected_ophandle(132))
4336         d.addCallback(lambda ign:
4337             self.clock.advance(96*60*60))
4338         d.addCallback(lambda ign:
4339             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4340                                  404, "404 Not Found",
4341                                  "unknown/expired handle '132'",
4342                                  self.GET,
4343                                  "/operations/132?t=status&output=JSON"))
4344         return d
4345
4346     def test_collected_ophandle_expiration(self):
4347         # collected ophandles should expire after 1 day
4348         def _make_collected_ophandle(ophandle):
4349             d = self.POST(self.public_url +
4350                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4351                           followRedirect=True)
4352             # By following the initial redirect, we collect the ophandle
4353             # we've just created.
4354             return d
4355         # Create a collected ophandle, then collect it after 23 hours
4356         # and 59 seconds to make sure that it is still there.
4357         d = _make_collected_ophandle(133)
4358         d.addCallback(lambda ign:
4359             self.clock.advance((24*60*60) - 1))
4360         d.addCallback(lambda ign:
4361             self.GET("/operations/133?t=status&output=JSON"))
4362         def _check1(res):
4363             data = simplejson.loads(res)
4364             self.failUnless("finished" in data, res)
4365         d.addCallback(_check1)
4366         # Create another uncollected ophandle, then try to collect it
4367         # after 24 hours to make sure that it is gone.
4368         d.addCallback(lambda ign:
4369             _make_collected_ophandle(134))
4370         d.addCallback(lambda ign:
4371             self.clock.advance(24*60*60))
4372         d.addCallback(lambda ign:
4373             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4374                                  404, "404 Not Found",
4375                                  "unknown/expired handle '134'",
4376                                  self.GET,
4377                                  "/operations/134?t=status&output=JSON"))
4378         return d
4379
4380     def test_incident(self):
4381         d = self.POST("/report_incident", details="eek")
4382         def _done(res):
4383             self.failIfIn("<html>", res)
4384             self.failUnlessIn("An incident report has been saved", res)
4385         d.addCallback(_done)
4386         return d
4387
4388     def test_static(self):
4389         webdir = os.path.join(self.staticdir, "subdir")
4390         fileutil.make_dirs(webdir)
4391         f = open(os.path.join(webdir, "hello.txt"), "wb")
4392         f.write("hello")
4393         f.close()
4394
4395         d = self.GET("/static/subdir/hello.txt")
4396         def _check(res):
4397             self.failUnlessReallyEqual(res, "hello")
4398         d.addCallback(_check)
4399         return d
4400
4401
4402 class IntroducerWeb(unittest.TestCase):
4403     def setUp(self):
4404         self.node = None
4405
4406     def tearDown(self):
4407         d = defer.succeed(None)
4408         if self.node:
4409             d.addCallback(lambda ign: self.node.stopService())
4410         d.addCallback(flushEventualQueue)
4411         return d
4412
4413     def test_welcome(self):
4414         basedir = "web.IntroducerWeb.test_welcome"
4415         os.mkdir(basedir)
4416         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4417         self.node = IntroducerNode(basedir)
4418         self.ws = self.node.getServiceNamed("webish")
4419
4420         d = fireEventually(None)
4421         d.addCallback(lambda ign: self.node.startService())
4422         d.addCallback(lambda ign: self.node.when_tub_ready())
4423
4424         d.addCallback(lambda ign: self.GET("/"))
4425         def _check(res):
4426             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4427             self.failUnlessIn(FAVICON_MARKUP, res)
4428             self.failUnlessIn('Page rendered at', res)
4429             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4430         d.addCallback(_check)
4431         return d
4432
4433     def GET(self, urlpath, followRedirect=False, return_response=False,
4434             **kwargs):
4435         # if return_response=True, this fires with (data, statuscode,
4436         # respheaders) instead of just data.
4437         assert not isinstance(urlpath, unicode)
4438         url = self.ws.getURL().rstrip('/') + urlpath
4439         factory = HTTPClientGETFactory(url, method="GET",
4440                                        followRedirect=followRedirect, **kwargs)
4441         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4442         d = factory.deferred
4443         def _got_data(data):
4444             return (data, factory.status, factory.response_headers)
4445         if return_response:
4446             d.addCallback(_got_data)
4447         return factory.deferred
4448
4449
4450 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4451     def test_load_file(self):
4452         # This will raise an exception unless a well-formed XML file is found under that name.
4453         common.getxmlfile('directory.xhtml').load()
4454
4455     def test_parse_replace_arg(self):
4456         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4457         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4458         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4459                                    "only-files")
4460         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4461
4462     def test_abbreviate_time(self):
4463         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4464         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4465         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4466         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4467         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4468         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4469
4470     def test_compute_rate(self):
4471         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4472         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4473         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4474         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4475         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4476         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4477         self.shouldFail(AssertionError, "test_compute_rate", "",
4478                         common.compute_rate, -100, 10)
4479         self.shouldFail(AssertionError, "test_compute_rate", "",
4480                         common.compute_rate, 100, -10)
4481
4482         # Sanity check
4483         rate = common.compute_rate(10*1000*1000, 1)
4484         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4485
4486     def test_abbreviate_rate(self):
4487         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4488         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4489         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4490         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4491
4492     def test_abbreviate_size(self):
4493         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4494         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4495         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4496         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4497         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4498
4499     def test_plural(self):
4500         def convert(s):
4501             return "%d second%s" % (s, status.plural(s))
4502         self.failUnlessReallyEqual(convert(0), "0 seconds")
4503         self.failUnlessReallyEqual(convert(1), "1 second")
4504         self.failUnlessReallyEqual(convert(2), "2 seconds")
4505         def convert2(s):
4506             return "has share%s: %s" % (status.plural(s), ",".join(s))
4507         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4508         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4509         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4510
4511
4512 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4513
4514     def CHECK(self, ign, which, args, clientnum=0):
4515         fileurl = self.fileurls[which]
4516         url = fileurl + "?" + args
4517         return self.GET(url, method="POST", clientnum=clientnum)
4518
4519     def test_filecheck(self):
4520         self.basedir = "web/Grid/filecheck"
4521         self.set_up_grid()
4522         c0 = self.g.clients[0]
4523         self.uris = {}
4524         DATA = "data" * 100
4525         d = c0.upload(upload.Data(DATA, convergence=""))
4526         def _stash_uri(ur, which):
4527             self.uris[which] = ur.get_uri()
4528         d.addCallback(_stash_uri, "good")
4529         d.addCallback(lambda ign:
4530                       c0.upload(upload.Data(DATA+"1", convergence="")))
4531         d.addCallback(_stash_uri, "sick")
4532         d.addCallback(lambda ign:
4533                       c0.upload(upload.Data(DATA+"2", convergence="")))
4534         d.addCallback(_stash_uri, "dead")
4535         def _stash_mutable_uri(n, which):
4536             self.uris[which] = n.get_uri()
4537             assert isinstance(self.uris[which], str)
4538         d.addCallback(lambda ign:
4539             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4540         d.addCallback(_stash_mutable_uri, "corrupt")
4541         d.addCallback(lambda ign:
4542                       c0.upload(upload.Data("literal", convergence="")))
4543         d.addCallback(_stash_uri, "small")
4544         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4545         d.addCallback(_stash_mutable_uri, "smalldir")
4546
4547         def _compute_fileurls(ignored):
4548             self.fileurls = {}
4549             for which in self.uris:
4550                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4551         d.addCallback(_compute_fileurls)
4552
4553         def _clobber_shares(ignored):
4554             good_shares = self.find_uri_shares(self.uris["good"])
4555             self.failUnlessReallyEqual(len(good_shares), 10)
4556             sick_shares = self.find_uri_shares(self.uris["sick"])
4557             os.unlink(sick_shares[0][2])
4558             dead_shares = self.find_uri_shares(self.uris["dead"])
4559             for i in range(1, 10):
4560                 os.unlink(dead_shares[i][2])
4561             c_shares = self.find_uri_shares(self.uris["corrupt"])
4562             cso = CorruptShareOptions()
4563             cso.stdout = StringIO()
4564             cso.parseOptions([c_shares[0][2]])
4565             corrupt_share(cso)
4566         d.addCallback(_clobber_shares)
4567
4568         d.addCallback(self.CHECK, "good", "t=check")
4569         def _got_html_good(res):
4570             self.failUnlessIn("Healthy", res)
4571             self.failIfIn("Not Healthy", res)
4572             self.failUnlessIn(FAVICON_MARKUP, res)
4573         d.addCallback(_got_html_good)
4574         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4575         def _got_html_good_return_to(res):
4576             self.failUnlessIn("Healthy", res)
4577             self.failIfIn("Not Healthy", res)
4578             self.failUnlessIn('<a href="somewhere">Return to file', res)
4579         d.addCallback(_got_html_good_return_to)
4580         d.addCallback(self.CHECK, "good", "t=check&output=json")
4581         def _got_json_good(res):
4582             r = simplejson.loads(res)
4583             self.failUnlessEqual(r["summary"], "Healthy")
4584             self.failUnless(r["results"]["healthy"])
4585             self.failIfIn("needs-rebalancing", r["results"])
4586             self.failUnless(r["results"]["recoverable"])
4587         d.addCallback(_got_json_good)
4588
4589         d.addCallback(self.CHECK, "small", "t=check")
4590         def _got_html_small(res):
4591             self.failUnlessIn("Literal files are always healthy", res)
4592             self.failIfIn("Not Healthy", res)
4593         d.addCallback(_got_html_small)
4594         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4595         def _got_html_small_return_to(res):
4596             self.failUnlessIn("Literal files are always healthy", res)
4597             self.failIfIn("Not Healthy", res)
4598             self.failUnlessIn('<a href="somewhere">Return to file', res)
4599         d.addCallback(_got_html_small_return_to)
4600         d.addCallback(self.CHECK, "small", "t=check&output=json")
4601         def _got_json_small(res):
4602             r = simplejson.loads(res)
4603             self.failUnlessEqual(r["storage-index"], "")
4604             self.failUnless(r["results"]["healthy"])
4605         d.addCallback(_got_json_small)
4606
4607         d.addCallback(self.CHECK, "smalldir", "t=check")
4608         def _got_html_smalldir(res):
4609             self.failUnlessIn("Literal files are always healthy", res)
4610             self.failIfIn("Not Healthy", res)
4611         d.addCallback(_got_html_smalldir)
4612         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4613         def _got_json_smalldir(res):
4614             r = simplejson.loads(res)
4615             self.failUnlessEqual(r["storage-index"], "")
4616             self.failUnless(r["results"]["healthy"])
4617         d.addCallback(_got_json_smalldir)
4618
4619         d.addCallback(self.CHECK, "sick", "t=check")
4620         def _got_html_sick(res):
4621             self.failUnlessIn("Not Healthy", res)
4622         d.addCallback(_got_html_sick)
4623         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4624         def _got_json_sick(res):
4625             r = simplejson.loads(res)
4626             self.failUnlessEqual(r["summary"],
4627                                  "Not Healthy: 9 shares (enc 3-of-10)")
4628             self.failIf(r["results"]["healthy"])
4629             self.failUnless(r["results"]["recoverable"])
4630             self.failIfIn("needs-rebalancing", r["results"])
4631         d.addCallback(_got_json_sick)
4632
4633         d.addCallback(self.CHECK, "dead", "t=check")
4634         def _got_html_dead(res):
4635             self.failUnlessIn("Not Healthy", res)
4636         d.addCallback(_got_html_dead)
4637         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4638         def _got_json_dead(res):
4639             r = simplejson.loads(res)
4640             self.failUnlessEqual(r["summary"],
4641                                  "Not Healthy: 1 shares (enc 3-of-10)")
4642             self.failIf(r["results"]["healthy"])
4643             self.failIf(r["results"]["recoverable"])
4644             self.failIfIn("needs-rebalancing", r["results"])
4645         d.addCallback(_got_json_dead)
4646
4647         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4648         def _got_html_corrupt(res):
4649             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4650         d.addCallback(_got_html_corrupt)
4651         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4652         def _got_json_corrupt(res):
4653             r = simplejson.loads(res)
4654             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4655             self.failIf(r["results"]["healthy"])
4656             self.failUnless(r["results"]["recoverable"])
4657             self.failIfIn("needs-rebalancing", r["results"])
4658             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4659             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4660             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4661         d.addCallback(_got_json_corrupt)
4662
4663         d.addErrback(self.explain_web_error)
4664         return d
4665
4666     def test_repair_html(self):
4667         self.basedir = "web/Grid/repair_html"
4668         self.set_up_grid()
4669         c0 = self.g.clients[0]
4670         self.uris = {}
4671         DATA = "data" * 100
4672         d = c0.upload(upload.Data(DATA, convergence=""))
4673         def _stash_uri(ur, which):
4674             self.uris[which] = ur.get_uri()
4675         d.addCallback(_stash_uri, "good")
4676         d.addCallback(lambda ign:
4677                       c0.upload(upload.Data(DATA+"1", convergence="")))
4678         d.addCallback(_stash_uri, "sick")
4679         d.addCallback(lambda ign:
4680                       c0.upload(upload.Data(DATA+"2", convergence="")))
4681         d.addCallback(_stash_uri, "dead")
4682         def _stash_mutable_uri(n, which):
4683             self.uris[which] = n.get_uri()
4684             assert isinstance(self.uris[which], str)
4685         d.addCallback(lambda ign:
4686             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4687         d.addCallback(_stash_mutable_uri, "corrupt")
4688
4689         def _compute_fileurls(ignored):
4690             self.fileurls = {}
4691             for which in self.uris:
4692                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4693         d.addCallback(_compute_fileurls)
4694
4695         def _clobber_shares(ignored):
4696             good_shares = self.find_uri_shares(self.uris["good"])
4697             self.failUnlessReallyEqual(len(good_shares), 10)
4698             sick_shares = self.find_uri_shares(self.uris["sick"])
4699             os.unlink(sick_shares[0][2])
4700             dead_shares = self.find_uri_shares(self.uris["dead"])
4701             for i in range(1, 10):
4702                 os.unlink(dead_shares[i][2])
4703             c_shares = self.find_uri_shares(self.uris["corrupt"])
4704             cso = CorruptShareOptions()
4705             cso.stdout = StringIO()
4706             cso.parseOptions([c_shares[0][2]])
4707             corrupt_share(cso)
4708         d.addCallback(_clobber_shares)
4709
4710         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4711         def _got_html_good(res):
4712             self.failUnlessIn("Healthy", res)
4713             self.failIfIn("Not Healthy", res)
4714             self.failUnlessIn("No repair necessary", res)
4715             self.failUnlessIn(FAVICON_MARKUP, res)
4716         d.addCallback(_got_html_good)
4717
4718         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4719         def _got_html_sick(res):
4720             self.failUnlessIn("Healthy : healthy", res)
4721             self.failIfIn("Not Healthy", res)
4722             self.failUnlessIn("Repair successful", res)
4723         d.addCallback(_got_html_sick)
4724
4725         # repair of a dead file will fail, of course, but it isn't yet
4726         # clear how this should be reported. Right now it shows up as
4727         # a "410 Gone".
4728         #
4729         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4730         #def _got_html_dead(res):
4731         #    print res
4732         #    self.failUnlessIn("Healthy : healthy", res)
4733         #    self.failIfIn("Not Healthy", res)
4734         #    self.failUnlessIn("No repair necessary", res)
4735         #d.addCallback(_got_html_dead)
4736
4737         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4738         def _got_html_corrupt(res):
4739             self.failUnlessIn("Healthy : Healthy", res)
4740             self.failIfIn("Not Healthy", res)
4741             self.failUnlessIn("Repair successful", res)
4742         d.addCallback(_got_html_corrupt)
4743
4744         d.addErrback(self.explain_web_error)
4745         return d
4746
4747     def test_repair_json(self):
4748         self.basedir = "web/Grid/repair_json"
4749         self.set_up_grid()
4750         c0 = self.g.clients[0]
4751         self.uris = {}
4752         DATA = "data" * 100
4753         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4754         def _stash_uri(ur, which):
4755             self.uris[which] = ur.get_uri()
4756         d.addCallback(_stash_uri, "sick")
4757
4758         def _compute_fileurls(ignored):
4759             self.fileurls = {}
4760             for which in self.uris:
4761                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4762         d.addCallback(_compute_fileurls)
4763
4764         def _clobber_shares(ignored):
4765             sick_shares = self.find_uri_shares(self.uris["sick"])
4766             os.unlink(sick_shares[0][2])
4767         d.addCallback(_clobber_shares)
4768
4769         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4770         def _got_json_sick(res):
4771             r = simplejson.loads(res)
4772             self.failUnlessReallyEqual(r["repair-attempted"], True)
4773             self.failUnlessReallyEqual(r["repair-successful"], True)
4774             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4775                                  "Not Healthy: 9 shares (enc 3-of-10)")
4776             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4777             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4778             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4779         d.addCallback(_got_json_sick)
4780
4781         d.addErrback(self.explain_web_error)
4782         return d
4783
4784     def test_unknown(self, immutable=False):
4785         self.basedir = "web/Grid/unknown"
4786         if immutable:
4787             self.basedir = "web/Grid/unknown-immutable"
4788
4789         self.set_up_grid()
4790         c0 = self.g.clients[0]
4791         self.uris = {}
4792         self.fileurls = {}
4793
4794         # the future cap format may contain slashes, which must be tolerated
4795         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4796                                                            safe="")
4797
4798         if immutable:
4799             name = u"future-imm"
4800             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4801             d = c0.create_immutable_dirnode({name: (future_node, {})})
4802         else:
4803             name = u"future"
4804             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4805             d = c0.create_dirnode()
4806
4807         def _stash_root_and_create_file(n):
4808             self.rootnode = n
4809             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4810             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4811             if not immutable:
4812                 return self.rootnode.set_node(name, future_node)
4813         d.addCallback(_stash_root_and_create_file)
4814
4815         # make sure directory listing tolerates unknown nodes
4816         d.addCallback(lambda ign: self.GET(self.rooturl))
4817         def _check_directory_html(res, expected_type_suffix):
4818             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4819                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4820                                  re.DOTALL)
4821             self.failUnless(re.search(pattern, res), res)
4822             # find the More Info link for name, should be relative
4823             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4824             info_url = mo.group(1)
4825             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4826         if immutable:
4827             d.addCallback(_check_directory_html, "-IMM")
4828         else:
4829             d.addCallback(_check_directory_html, "")
4830
4831         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4832         def _check_directory_json(res, expect_rw_uri):
4833             data = simplejson.loads(res)
4834             self.failUnlessEqual(data[0], "dirnode")
4835             f = data[1]["children"][name]
4836             self.failUnlessEqual(f[0], "unknown")
4837             if expect_rw_uri:
4838                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4839             else:
4840                 self.failIfIn("rw_uri", f[1])
4841             if immutable:
4842                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4843             else:
4844                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4845             self.failUnlessIn("metadata", f[1])
4846         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4847
4848         def _check_info(res, expect_rw_uri, expect_ro_uri):
4849             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4850             if expect_rw_uri:
4851                 self.failUnlessIn(unknown_rwcap, res)
4852             if expect_ro_uri:
4853                 if immutable:
4854                     self.failUnlessIn(unknown_immcap, res)
4855                 else:
4856                     self.failUnlessIn(unknown_rocap, res)
4857             else:
4858                 self.failIfIn(unknown_rocap, res)
4859             self.failIfIn("Raw data as", res)
4860             self.failIfIn("Directory writecap", res)
4861             self.failIfIn("Checker Operations", res)
4862             self.failIfIn("Mutable File Operations", res)
4863             self.failIfIn("Directory Operations", res)
4864
4865         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4866         # why they fail. Possibly related to ticket #922.
4867
4868         d.addCallback(lambda ign: self.GET(expected_info_url))
4869         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4870         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4871         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4872
4873         def _check_json(res, expect_rw_uri):
4874             data = simplejson.loads(res)
4875             self.failUnlessEqual(data[0], "unknown")
4876             if expect_rw_uri:
4877                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4878             else:
4879                 self.failIfIn("rw_uri", data[1])
4880
4881             if immutable:
4882                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4883                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4884             elif expect_rw_uri:
4885                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4886                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4887             else:
4888                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4889                 self.failIfIn("mutable", data[1])
4890
4891             # TODO: check metadata contents
4892             self.failUnlessIn("metadata", data[1])
4893
4894         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4895         d.addCallback(_check_json, expect_rw_uri=not immutable)
4896
4897         # and make sure that a read-only version of the directory can be
4898         # rendered too. This version will not have unknown_rwcap, whether
4899         # or not future_node was immutable.
4900         d.addCallback(lambda ign: self.GET(self.rourl))
4901         if immutable:
4902             d.addCallback(_check_directory_html, "-IMM")
4903         else:
4904             d.addCallback(_check_directory_html, "-RO")
4905
4906         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4907         d.addCallback(_check_directory_json, expect_rw_uri=False)
4908
4909         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4910         d.addCallback(_check_json, expect_rw_uri=False)
4911
4912         # TODO: check that getting t=info from the Info link in the ro directory
4913         # works, and does not include the writecap URI.
4914         return d
4915
4916     def test_immutable_unknown(self):
4917         return self.test_unknown(immutable=True)
4918
4919     def test_mutant_dirnodes_are_omitted(self):
4920         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4921
4922         self.set_up_grid()
4923         c = self.g.clients[0]
4924         nm = c.nodemaker
4925         self.uris = {}
4926         self.fileurls = {}
4927
4928         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4929         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4930         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4931
4932         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4933         # test the dirnode and web layers separately.
4934
4935         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4936         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4937         # When the directory is read, the mutants should be silently disposed of, leaving
4938         # their lonely sibling.
4939         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4940         # because immutable directories don't have a writecap and therefore that field
4941         # isn't (and can't be) decrypted.
4942         # TODO: The field still exists in the netstring. Technically we should check what
4943         # happens if something is put there (_unpack_contents should raise ValueError),
4944         # but that can wait.
4945
4946         lonely_child = nm.create_from_cap(lonely_uri)
4947         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4948         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4949
4950         def _by_hook_or_by_crook():
4951             return True
4952         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4953             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4954
4955         mutant_write_in_ro_child.get_write_uri    = lambda: None
4956         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4957
4958         kids = {u"lonely":      (lonely_child, {}),
4959                 u"ro":          (mutant_ro_child, {}),
4960                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4961                 }
4962         d = c.create_immutable_dirnode(kids)
4963
4964         def _created(dn):
4965             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4966             self.failIf(dn.is_mutable())
4967             self.failUnless(dn.is_readonly())
4968             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4969             self.failIf(hasattr(dn._node, 'get_writekey'))
4970             rep = str(dn)
4971             self.failUnlessIn("RO-IMM", rep)
4972             cap = dn.get_cap()
4973             self.failUnlessIn("CHK", cap.to_string())
4974             self.cap = cap
4975             self.rootnode = dn
4976             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4977             return download_to_data(dn._node)
4978         d.addCallback(_created)
4979
4980         def _check_data(data):
4981             # Decode the netstring representation of the directory to check that all children
4982             # are present. This is a bit of an abstraction violation, but there's not really
4983             # any other way to do it given that the real DirectoryNode._unpack_contents would
4984             # strip the mutant children out (which is what we're trying to test, later).
4985             position = 0
4986             numkids = 0
4987             while position < len(data):
4988                 entries, position = split_netstring(data, 1, position)
4989                 entry = entries[0]
4990                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4991                 name = name_utf8.decode("utf-8")
4992                 self.failUnlessEqual(rwcapdata, "")
4993                 self.failUnlessIn(name, kids)
4994                 (expected_child, ign) = kids[name]
4995                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4996                 numkids += 1
4997
4998             self.failUnlessReallyEqual(numkids, 3)
4999             return self.rootnode.list()
5000         d.addCallback(_check_data)
5001
5002         # Now when we use the real directory listing code, the mutants should be absent.
5003         def _check_kids(children):
5004             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5005             lonely_node, lonely_metadata = children[u"lonely"]
5006
5007             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5008             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5009         d.addCallback(_check_kids)
5010
5011         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5012         d.addCallback(lambda n: n.list())
5013         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5014
5015         # Make sure the lonely child can be listed in HTML...
5016         d.addCallback(lambda ign: self.GET(self.rooturl))
5017         def _check_html(res):
5018             self.failIfIn("URI:SSK", res)
5019             get_lonely = "".join([r'<td>FILE</td>',
5020                                   r'\s+<td>',
5021                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5022                                   r'</td>',
5023                                   r'\s+<td align="right">%d</td>' % len("one"),
5024                                   ])
5025             self.failUnless(re.search(get_lonely, res), res)
5026
5027             # find the More Info link for name, should be relative
5028             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5029             info_url = mo.group(1)
5030             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5031         d.addCallback(_check_html)
5032
5033         # ... and in JSON.
5034         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5035         def _check_json(res):
5036             data = simplejson.loads(res)
5037             self.failUnlessEqual(data[0], "dirnode")
5038             listed_children = data[1]["children"]
5039             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5040             ll_type, ll_data = listed_children[u"lonely"]
5041             self.failUnlessEqual(ll_type, "filenode")
5042             self.failIfIn("rw_uri", ll_data)
5043             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5044         d.addCallback(_check_json)
5045         return d
5046
5047     def test_deep_check(self):
5048         self.basedir = "web/Grid/deep_check"
5049         self.set_up_grid()
5050         c0 = self.g.clients[0]
5051         self.uris = {}
5052         self.fileurls = {}
5053         DATA = "data" * 100
5054         d = c0.create_dirnode()
5055         def _stash_root_and_create_file(n):
5056             self.rootnode = n
5057             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5058             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5059         d.addCallback(_stash_root_and_create_file)
5060         def _stash_uri(fn, which):
5061             self.uris[which] = fn.get_uri()
5062             return fn
5063         d.addCallback(_stash_uri, "good")
5064         d.addCallback(lambda ign:
5065                       self.rootnode.add_file(u"small",
5066                                              upload.Data("literal",
5067                                                         convergence="")))
5068         d.addCallback(_stash_uri, "small")
5069         d.addCallback(lambda ign:
5070                       self.rootnode.add_file(u"sick",
5071                                              upload.Data(DATA+"1",
5072                                                         convergence="")))
5073         d.addCallback(_stash_uri, "sick")
5074
5075         # this tests that deep-check and stream-manifest will ignore
5076         # UnknownNode instances. Hopefully this will also cover deep-stats.
5077         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5078         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5079
5080         def _clobber_shares(ignored):
5081             self.delete_shares_numbered(self.uris["sick"], [0,1])
5082         d.addCallback(_clobber_shares)
5083
5084         # root
5085         # root/good
5086         # root/small
5087         # root/sick
5088         # root/future
5089
5090         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5091         def _done(res):
5092             try:
5093                 units = [simplejson.loads(line)
5094                          for line in res.splitlines()
5095                          if line]
5096             except ValueError:
5097                 print "response is:", res
5098                 print "undecodeable line was '%s'" % line
5099                 raise
5100             self.failUnlessReallyEqual(len(units), 5+1)
5101             # should be parent-first
5102             u0 = units[0]
5103             self.failUnlessEqual(u0["path"], [])
5104             self.failUnlessEqual(u0["type"], "directory")
5105             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5106             u0cr = u0["check-results"]
5107             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5108             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5109
5110             ugood = [u for u in units
5111                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5112             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5113             ugoodcr = ugood["check-results"]
5114             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5115             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5116
5117             stats = units[-1]
5118             self.failUnlessEqual(stats["type"], "stats")
5119             s = stats["stats"]
5120             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5121             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5122             self.failUnlessReallyEqual(s["count-directories"], 1)
5123             self.failUnlessReallyEqual(s["count-unknown"], 1)
5124         d.addCallback(_done)
5125
5126         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5127         def _check_manifest(res):
5128             self.failUnless(res.endswith("\n"))
5129             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5130             self.failUnlessReallyEqual(len(units), 5+1)
5131             self.failUnlessEqual(units[-1]["type"], "stats")
5132             first = units[0]
5133             self.failUnlessEqual(first["path"], [])
5134             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5135             self.failUnlessEqual(first["type"], "directory")
5136             stats = units[-1]["stats"]
5137             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5138             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5139             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5140             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5141             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5142         d.addCallback(_check_manifest)
5143
5144         # now add root/subdir and root/subdir/grandchild, then make subdir
5145         # unrecoverable, then see what happens
5146
5147         d.addCallback(lambda ign:
5148                       self.rootnode.create_subdirectory(u"subdir"))
5149         d.addCallback(_stash_uri, "subdir")
5150         d.addCallback(lambda subdir_node:
5151                       subdir_node.add_file(u"grandchild",
5152                                            upload.Data(DATA+"2",
5153                                                        convergence="")))
5154         d.addCallback(_stash_uri, "grandchild")
5155
5156         d.addCallback(lambda ign:
5157                       self.delete_shares_numbered(self.uris["subdir"],
5158                                                   range(1, 10)))
5159
5160         # root
5161         # root/good
5162         # root/small
5163         # root/sick
5164         # root/future
5165         # root/subdir [unrecoverable]
5166         # root/subdir/grandchild
5167
5168         # how should a streaming-JSON API indicate fatal error?
5169         # answer: emit ERROR: instead of a JSON string
5170
5171         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5172         def _check_broken_manifest(res):
5173             lines = res.splitlines()
5174             error_lines = [i
5175                            for (i,line) in enumerate(lines)
5176                            if line.startswith("ERROR:")]
5177             if not error_lines:
5178                 self.fail("no ERROR: in output: %s" % (res,))
5179             first_error = error_lines[0]
5180             error_line = lines[first_error]
5181             error_msg = lines[first_error+1:]
5182             error_msg_s = "\n".join(error_msg) + "\n"
5183             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5184                               error_line)
5185             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5186             units = [simplejson.loads(line) for line in lines[:first_error]]
5187             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5188             last_unit = units[-1]
5189             self.failUnlessEqual(last_unit["path"], ["subdir"])
5190         d.addCallback(_check_broken_manifest)
5191
5192         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5193         def _check_broken_deepcheck(res):
5194             lines = res.splitlines()
5195             error_lines = [i
5196                            for (i,line) in enumerate(lines)
5197                            if line.startswith("ERROR:")]
5198             if not error_lines:
5199                 self.fail("no ERROR: in output: %s" % (res,))
5200             first_error = error_lines[0]
5201             error_line = lines[first_error]
5202             error_msg = lines[first_error+1:]
5203             error_msg_s = "\n".join(error_msg) + "\n"
5204             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5205                               error_line)
5206             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5207             units = [simplejson.loads(line) for line in lines[:first_error]]
5208             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5209             last_unit = units[-1]
5210             self.failUnlessEqual(last_unit["path"], ["subdir"])
5211             r = last_unit["check-results"]["results"]
5212             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5213             self.failUnlessReallyEqual(r["count-happiness"], 1)
5214             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5215             self.failUnlessReallyEqual(r["recoverable"], False)
5216         d.addCallback(_check_broken_deepcheck)
5217
5218         d.addErrback(self.explain_web_error)
5219         return d
5220
5221     def test_deep_check_and_repair(self):
5222         self.basedir = "web/Grid/deep_check_and_repair"
5223         self.set_up_grid()
5224         c0 = self.g.clients[0]
5225         self.uris = {}
5226         self.fileurls = {}
5227         DATA = "data" * 100
5228         d = c0.create_dirnode()
5229         def _stash_root_and_create_file(n):
5230             self.rootnode = n
5231             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5232             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5233         d.addCallback(_stash_root_and_create_file)
5234         def _stash_uri(fn, which):
5235             self.uris[which] = fn.get_uri()
5236         d.addCallback(_stash_uri, "good")
5237         d.addCallback(lambda ign:
5238                       self.rootnode.add_file(u"small",
5239                                              upload.Data("literal",
5240                                                         convergence="")))
5241         d.addCallback(_stash_uri, "small")
5242         d.addCallback(lambda ign:
5243                       self.rootnode.add_file(u"sick",
5244                                              upload.Data(DATA+"1",
5245                                                         convergence="")))
5246         d.addCallback(_stash_uri, "sick")
5247         #d.addCallback(lambda ign:
5248         #              self.rootnode.add_file(u"dead",
5249         #                                     upload.Data(DATA+"2",
5250         #                                                convergence="")))
5251         #d.addCallback(_stash_uri, "dead")
5252
5253         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5254         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5255         #d.addCallback(_stash_uri, "corrupt")
5256
5257         def _clobber_shares(ignored):
5258             good_shares = self.find_uri_shares(self.uris["good"])
5259             self.failUnlessReallyEqual(len(good_shares), 10)
5260             sick_shares = self.find_uri_shares(self.uris["sick"])
5261             os.unlink(sick_shares[0][2])
5262             #dead_shares = self.find_uri_shares(self.uris["dead"])
5263             #for i in range(1, 10):
5264             #    os.unlink(dead_shares[i][2])
5265
5266             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5267             #cso = CorruptShareOptions()
5268             #cso.stdout = StringIO()
5269             #cso.parseOptions([c_shares[0][2]])
5270             #corrupt_share(cso)
5271         d.addCallback(_clobber_shares)
5272
5273         # root
5274         # root/good   CHK, 10 shares
5275         # root/small  LIT
5276         # root/sick   CHK, 9 shares
5277
5278         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5279         def _done(res):
5280             units = [simplejson.loads(line)
5281                      for line in res.splitlines()
5282                      if line]
5283             self.failUnlessReallyEqual(len(units), 4+1)
5284             # should be parent-first
5285             u0 = units[0]
5286             self.failUnlessEqual(u0["path"], [])
5287             self.failUnlessEqual(u0["type"], "directory")
5288             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5289             u0crr = u0["check-and-repair-results"]
5290             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5291             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5292             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5293
5294             ugood = [u for u in units
5295                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5296             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5297             ugoodcrr = ugood["check-and-repair-results"]
5298             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5299             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5300             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5301
5302             usick = [u for u in units
5303                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5304             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5305             usickcrr = usick["check-and-repair-results"]
5306             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5307             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5308             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5309             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5310             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5311             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5312
5313             stats = units[-1]
5314             self.failUnlessEqual(stats["type"], "stats")
5315             s = stats["stats"]
5316             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5317             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5318             self.failUnlessReallyEqual(s["count-directories"], 1)
5319         d.addCallback(_done)
5320
5321         d.addErrback(self.explain_web_error)
5322         return d
5323
5324     def _count_leases(self, ignored, which):
5325         u = self.uris[which]
5326         shares = self.find_uri_shares(u)
5327         lease_counts = []
5328         for shnum, serverid, fn in shares:
5329             sf = get_share_file(fn)
5330             num_leases = len(list(sf.get_leases()))
5331             lease_counts.append( (fn, num_leases) )
5332         return lease_counts
5333
5334     def _assert_leasecount(self, lease_counts, expected):
5335         for (fn, num_leases) in lease_counts:
5336             if num_leases != expected:
5337                 self.fail("expected %d leases, have %d, on %s" %
5338                           (expected, num_leases, fn))
5339
5340     def test_add_lease(self):
5341         self.basedir = "web/Grid/add_lease"
5342         self.set_up_grid(num_clients=2)
5343         c0 = self.g.clients[0]
5344         self.uris = {}
5345         DATA = "data" * 100
5346         d = c0.upload(upload.Data(DATA, convergence=""))
5347         def _stash_uri(ur, which):
5348             self.uris[which] = ur.get_uri()
5349         d.addCallback(_stash_uri, "one")
5350         d.addCallback(lambda ign:
5351                       c0.upload(upload.Data(DATA+"1", convergence="")))
5352         d.addCallback(_stash_uri, "two")
5353         def _stash_mutable_uri(n, which):
5354             self.uris[which] = n.get_uri()
5355             assert isinstance(self.uris[which], str)
5356         d.addCallback(lambda ign:
5357             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5358         d.addCallback(_stash_mutable_uri, "mutable")
5359
5360         def _compute_fileurls(ignored):
5361             self.fileurls = {}
5362             for which in self.uris:
5363                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5364         d.addCallback(_compute_fileurls)
5365
5366         d.addCallback(self._count_leases, "one")
5367         d.addCallback(self._assert_leasecount, 1)
5368         d.addCallback(self._count_leases, "two")
5369         d.addCallback(self._assert_leasecount, 1)
5370         d.addCallback(self._count_leases, "mutable")
5371         d.addCallback(self._assert_leasecount, 1)
5372
5373         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5374         def _got_html_good(res):
5375             self.failUnlessIn("Healthy", res)
5376             self.failIfIn("Not Healthy", res)
5377         d.addCallback(_got_html_good)
5378
5379         d.addCallback(self._count_leases, "one")
5380         d.addCallback(self._assert_leasecount, 1)
5381         d.addCallback(self._count_leases, "two")
5382         d.addCallback(self._assert_leasecount, 1)
5383         d.addCallback(self._count_leases, "mutable")
5384         d.addCallback(self._assert_leasecount, 1)
5385
5386         # this CHECK uses the original client, which uses the same
5387         # lease-secrets, so it will just renew the original lease
5388         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5389         d.addCallback(_got_html_good)
5390
5391         d.addCallback(self._count_leases, "one")
5392         d.addCallback(self._assert_leasecount, 1)
5393         d.addCallback(self._count_leases, "two")
5394         d.addCallback(self._assert_leasecount, 1)
5395         d.addCallback(self._count_leases, "mutable")
5396         d.addCallback(self._assert_leasecount, 1)
5397
5398         # this CHECK uses an alternate client, which adds a second lease
5399         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5400         d.addCallback(_got_html_good)
5401
5402         d.addCallback(self._count_leases, "one")
5403         d.addCallback(self._assert_leasecount, 2)
5404         d.addCallback(self._count_leases, "two")
5405         d.addCallback(self._assert_leasecount, 1)
5406         d.addCallback(self._count_leases, "mutable")
5407         d.addCallback(self._assert_leasecount, 1)
5408
5409         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5410         d.addCallback(_got_html_good)
5411
5412         d.addCallback(self._count_leases, "one")
5413         d.addCallback(self._assert_leasecount, 2)
5414         d.addCallback(self._count_leases, "two")
5415         d.addCallback(self._assert_leasecount, 1)
5416         d.addCallback(self._count_leases, "mutable")
5417         d.addCallback(self._assert_leasecount, 1)
5418
5419         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5420                       clientnum=1)
5421         d.addCallback(_got_html_good)
5422
5423         d.addCallback(self._count_leases, "one")
5424         d.addCallback(self._assert_leasecount, 2)
5425         d.addCallback(self._count_leases, "two")
5426         d.addCallback(self._assert_leasecount, 1)
5427         d.addCallback(self._count_leases, "mutable")
5428         d.addCallback(self._assert_leasecount, 2)
5429
5430         d.addErrback(self.explain_web_error)
5431         return d
5432
5433     def test_deep_add_lease(self):
5434         self.basedir = "web/Grid/deep_add_lease"
5435         self.set_up_grid(num_clients=2)
5436         c0 = self.g.clients[0]
5437         self.uris = {}
5438         self.fileurls = {}
5439         DATA = "data" * 100
5440         d = c0.create_dirnode()
5441         def _stash_root_and_create_file(n):
5442             self.rootnode = n
5443             self.uris["root"] = n.get_uri()
5444             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5445             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5446         d.addCallback(_stash_root_and_create_file)
5447         def _stash_uri(fn, which):
5448             self.uris[which] = fn.get_uri()
5449         d.addCallback(_stash_uri, "one")
5450         d.addCallback(lambda ign:
5451                       self.rootnode.add_file(u"small",
5452                                              upload.Data("literal",
5453                                                         convergence="")))
5454         d.addCallback(_stash_uri, "small")
5455
5456         d.addCallback(lambda ign:
5457             c0.create_mutable_file(publish.MutableData("mutable")))
5458         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5459         d.addCallback(_stash_uri, "mutable")
5460
5461         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5462         def _done(res):
5463             units = [simplejson.loads(line)
5464                      for line in res.splitlines()
5465                      if line]
5466             # root, one, small, mutable,   stats
5467             self.failUnlessReallyEqual(len(units), 4+1)
5468         d.addCallback(_done)
5469
5470         d.addCallback(self._count_leases, "root")
5471         d.addCallback(self._assert_leasecount, 1)
5472         d.addCallback(self._count_leases, "one")
5473         d.addCallback(self._assert_leasecount, 1)
5474         d.addCallback(self._count_leases, "mutable")
5475         d.addCallback(self._assert_leasecount, 1)
5476
5477         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5478         d.addCallback(_done)
5479
5480         d.addCallback(self._count_leases, "root")
5481         d.addCallback(self._assert_leasecount, 1)
5482         d.addCallback(self._count_leases, "one")
5483         d.addCallback(self._assert_leasecount, 1)
5484         d.addCallback(self._count_leases, "mutable")
5485         d.addCallback(self._assert_leasecount, 1)
5486
5487         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5488                       clientnum=1)
5489         d.addCallback(_done)
5490
5491         d.addCallback(self._count_leases, "root")
5492         d.addCallback(self._assert_leasecount, 2)
5493         d.addCallback(self._count_leases, "one")
5494         d.addCallback(self._assert_leasecount, 2)
5495         d.addCallback(self._count_leases, "mutable")
5496         d.addCallback(self._assert_leasecount, 2)
5497
5498         d.addErrback(self.explain_web_error)
5499         return d
5500
5501
5502     def test_exceptions(self):
5503         self.basedir = "web/Grid/exceptions"
5504         self.set_up_grid(num_clients=1, num_servers=2)
5505         c0 = self.g.clients[0]
5506         c0.encoding_params['happy'] = 2
5507         self.fileurls = {}
5508         DATA = "data" * 100
5509         d = c0.create_dirnode()
5510         def _stash_root(n):
5511             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5512             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5513             return n
5514         d.addCallback(_stash_root)
5515         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5516         def _stash_bad(ur):
5517             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5518             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5519
5520             u = uri.from_string(ur.get_uri())
5521             u.key = testutil.flip_bit(u.key, 0)
5522             baduri = u.to_string()
5523             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5524         d.addCallback(_stash_bad)
5525         d.addCallback(lambda ign: c0.create_dirnode())
5526         def _mangle_dirnode_1share(n):
5527             u = n.get_uri()
5528             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5529             self.fileurls["dir-1share-json"] = url + "?t=json"
5530             self.delete_shares_numbered(u, range(1,10))
5531         d.addCallback(_mangle_dirnode_1share)
5532         d.addCallback(lambda ign: c0.create_dirnode())
5533         def _mangle_dirnode_0share(n):
5534             u = n.get_uri()
5535             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5536             self.fileurls["dir-0share-json"] = url + "?t=json"
5537             self.delete_shares_numbered(u, range(0,10))
5538         d.addCallback(_mangle_dirnode_0share)
5539
5540         # NotEnoughSharesError should be reported sensibly, with a
5541         # text/plain explanation of the problem, and perhaps some
5542         # information on which shares *could* be found.
5543
5544         d.addCallback(lambda ignored:
5545                       self.shouldHTTPError("GET unrecoverable",
5546                                            410, "Gone", "NoSharesError",
5547                                            self.GET, self.fileurls["0shares"]))
5548         def _check_zero_shares(body):
5549             self.failIfIn("<html>", body)
5550             body = " ".join(body.strip().split())
5551             exp = ("NoSharesError: no shares could be found. "
5552                    "Zero shares usually indicates a corrupt URI, or that "
5553                    "no servers were connected, but it might also indicate "
5554                    "severe corruption. You should perform a filecheck on "
5555                    "this object to learn more. The full error message is: "
5556                    "no shares (need 3). Last failure: None")
5557             self.failUnlessReallyEqual(exp, body)
5558         d.addCallback(_check_zero_shares)
5559
5560
5561         d.addCallback(lambda ignored:
5562                       self.shouldHTTPError("GET 1share",
5563                                            410, "Gone", "NotEnoughSharesError",
5564                                            self.GET, self.fileurls["1share"]))
5565         def _check_one_share(body):
5566             self.failIfIn("<html>", body)
5567             body = " ".join(body.strip().split())
5568             msgbase = ("NotEnoughSharesError: This indicates that some "
5569                        "servers were unavailable, or that shares have been "
5570                        "lost to server departure, hard drive failure, or disk "
5571                        "corruption. You should perform a filecheck on "
5572                        "this object to learn more. The full error message is:"
5573                        )
5574             msg1 = msgbase + (" ran out of shares:"
5575                               " complete=sh0"
5576                               " pending="
5577                               " overdue= unused= need 3. Last failure: None")
5578             msg2 = msgbase + (" ran out of shares:"
5579                               " complete="
5580                               " pending=Share(sh0-on-xgru5)"
5581                               " overdue= unused= need 3. Last failure: None")
5582             self.failUnless(body == msg1 or body == msg2, body)
5583         d.addCallback(_check_one_share)
5584
5585         d.addCallback(lambda ignored:
5586                       self.shouldHTTPError("GET imaginary",
5587                                            404, "Not Found", None,
5588                                            self.GET, self.fileurls["imaginary"]))
5589         def _missing_child(body):
5590             self.failUnlessIn("No such child: imaginary", body)
5591         d.addCallback(_missing_child)
5592
5593         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5594         def _check_0shares_dir_html(body):
5595             self.failUnlessIn(DIR_HTML_TAG, body)
5596             # we should see the regular page, but without the child table or
5597             # the dirops forms
5598             body = " ".join(body.strip().split())
5599             self.failUnlessIn('href="?t=info">More info on this directory',
5600                               body)
5601             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5602                    "could not be retrieved, because there were insufficient "
5603                    "good shares. This might indicate that no servers were "
5604                    "connected, insufficient servers were connected, the URI "
5605                    "was corrupt, or that shares have been lost due to server "
5606                    "departure, hard drive failure, or disk corruption. You "
5607                    "should perform a filecheck on this object to learn more.")
5608             self.failUnlessIn(exp, body)
5609             self.failUnlessIn("No upload forms: directory is unreadable", body)
5610         d.addCallback(_check_0shares_dir_html)
5611
5612         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5613         def _check_1shares_dir_html(body):
5614             # at some point, we'll split UnrecoverableFileError into 0-shares
5615             # and some-shares like we did for immutable files (since there
5616             # are different sorts of advice to offer in each case). For now,
5617             # they present the same way.
5618             self.failUnlessIn(DIR_HTML_TAG, body)
5619             body = " ".join(body.strip().split())
5620             self.failUnlessIn('href="?t=info">More info on this directory',
5621                               body)
5622             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5623                    "could not be retrieved, because there were insufficient "
5624                    "good shares. This might indicate that no servers were "
5625                    "connected, insufficient servers were connected, the URI "
5626                    "was corrupt, or that shares have been lost due to server "
5627                    "departure, hard drive failure, or disk corruption. You "
5628                    "should perform a filecheck on this object to learn more.")
5629             self.failUnlessIn(exp, body)
5630             self.failUnlessIn("No upload forms: directory is unreadable", body)
5631         d.addCallback(_check_1shares_dir_html)
5632
5633         d.addCallback(lambda ignored:
5634                       self.shouldHTTPError("GET dir-0share-json",
5635                                            410, "Gone", "UnrecoverableFileError",
5636                                            self.GET,
5637                                            self.fileurls["dir-0share-json"]))
5638         def _check_unrecoverable_file(body):
5639             self.failIfIn("<html>", body)
5640             body = " ".join(body.strip().split())
5641             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5642                    "could not be retrieved, because there were insufficient "
5643                    "good shares. This might indicate that no servers were "
5644                    "connected, insufficient servers were connected, the URI "
5645                    "was corrupt, or that shares have been lost due to server "
5646                    "departure, hard drive failure, or disk corruption. You "
5647                    "should perform a filecheck on this object to learn more.")
5648             self.failUnlessReallyEqual(exp, body)
5649         d.addCallback(_check_unrecoverable_file)
5650
5651         d.addCallback(lambda ignored:
5652                       self.shouldHTTPError("GET dir-1share-json",
5653                                            410, "Gone", "UnrecoverableFileError",
5654                                            self.GET,
5655                                            self.fileurls["dir-1share-json"]))
5656         d.addCallback(_check_unrecoverable_file)
5657
5658         d.addCallback(lambda ignored:
5659                       self.shouldHTTPError("GET imaginary",
5660                                            404, "Not Found", None,
5661                                            self.GET, self.fileurls["imaginary"]))
5662
5663         # attach a webapi child that throws a random error, to test how it
5664         # gets rendered.
5665         w = c0.getServiceNamed("webish")
5666         w.root.putChild("ERRORBOOM", ErrorBoom())
5667
5668         # "Accept: */*" :        should get a text/html stack trace
5669         # "Accept: text/plain" : should get a text/plain stack trace
5670         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5671         # no Accept header:      should get a text/html stack trace
5672
5673         d.addCallback(lambda ignored:
5674                       self.shouldHTTPError("GET errorboom_html",
5675                                            500, "Internal Server Error", None,
5676                                            self.GET, "ERRORBOOM",
5677                                            headers={"accept": "*/*"}))
5678         def _internal_error_html1(body):
5679             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5680         d.addCallback(_internal_error_html1)
5681
5682         d.addCallback(lambda ignored:
5683                       self.shouldHTTPError("GET errorboom_text",
5684                                            500, "Internal Server Error", None,
5685                                            self.GET, "ERRORBOOM",
5686                                            headers={"accept": "text/plain"}))
5687         def _internal_error_text2(body):
5688             self.failIfIn("<html>", body)
5689             self.failUnless(body.startswith("Traceback "), body)
5690         d.addCallback(_internal_error_text2)
5691
5692         CLI_accepts = "text/plain, application/octet-stream"
5693         d.addCallback(lambda ignored:
5694                       self.shouldHTTPError("GET errorboom_text",
5695                                            500, "Internal Server Error", None,
5696                                            self.GET, "ERRORBOOM",
5697                                            headers={"accept": CLI_accepts}))
5698         def _internal_error_text3(body):
5699             self.failIfIn("<html>", body)
5700             self.failUnless(body.startswith("Traceback "), body)
5701         d.addCallback(_internal_error_text3)
5702
5703         d.addCallback(lambda ignored:
5704                       self.shouldHTTPError("GET errorboom_text",
5705                                            500, "Internal Server Error", None,
5706                                            self.GET, "ERRORBOOM"))
5707         def _internal_error_html4(body):
5708             self.failUnlessIn("<html>", body)
5709         d.addCallback(_internal_error_html4)
5710
5711         def _flush_errors(res):
5712             # Trial: please ignore the CompletelyUnhandledError in the logs
5713             self.flushLoggedErrors(CompletelyUnhandledError)
5714             return res
5715         d.addBoth(_flush_errors)
5716
5717         return d
5718
5719     def test_blacklist(self):
5720         # download from a blacklisted URI, get an error
5721         self.basedir = "web/Grid/blacklist"
5722         self.set_up_grid()
5723         c0 = self.g.clients[0]
5724         c0_basedir = c0.basedir
5725         fn = os.path.join(c0_basedir, "access.blacklist")
5726         self.uris = {}
5727         DATA = "off-limits " * 50
5728
5729         d = c0.upload(upload.Data(DATA, convergence=""))
5730         def _stash_uri_and_create_dir(ur):
5731             self.uri = ur.get_uri()
5732             self.url = "uri/"+self.uri
5733             u = uri.from_string_filenode(self.uri)
5734             self.si = u.get_storage_index()
5735             childnode = c0.create_node_from_uri(self.uri, None)
5736             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5737         d.addCallback(_stash_uri_and_create_dir)
5738         def _stash_dir(node):
5739             self.dir_node = node
5740             self.dir_uri = node.get_uri()
5741             self.dir_url = "uri/"+self.dir_uri
5742         d.addCallback(_stash_dir)
5743         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5744         def _check_dir_html(body):
5745             self.failUnlessIn(DIR_HTML_TAG, body)
5746             self.failUnlessIn("blacklisted.txt</a>", body)
5747         d.addCallback(_check_dir_html)
5748         d.addCallback(lambda ign: self.GET(self.url))
5749         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5750
5751         def _blacklist(ign):
5752             f = open(fn, "w")
5753             f.write(" # this is a comment\n")
5754             f.write(" \n")
5755             f.write("\n") # also exercise blank lines
5756             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5757             f.close()
5758             # clients should be checking the blacklist each time, so we don't
5759             # need to restart the client
5760         d.addCallback(_blacklist)
5761         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5762                                                        403, "Forbidden",
5763                                                        "Access Prohibited: off-limits",
5764                                                        self.GET, self.url))
5765
5766         # We should still be able to list the parent directory, in HTML...
5767         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5768         def _check_dir_html2(body):
5769             self.failUnlessIn(DIR_HTML_TAG, body)
5770             self.failUnlessIn("blacklisted.txt</strike>", body)
5771         d.addCallback(_check_dir_html2)
5772
5773         # ... and in JSON (used by CLI).
5774         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5775         def _check_dir_json(res):
5776             data = simplejson.loads(res)
5777             self.failUnless(isinstance(data, list), data)
5778             self.failUnlessEqual(data[0], "dirnode")
5779             self.failUnless(isinstance(data[1], dict), data)
5780             self.failUnlessIn("children", data[1])
5781             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5782             childdata = data[1]["children"]["blacklisted.txt"]
5783             self.failUnless(isinstance(childdata, list), data)
5784             self.failUnlessEqual(childdata[0], "filenode")
5785             self.failUnless(isinstance(childdata[1], dict), data)
5786         d.addCallback(_check_dir_json)
5787
5788         def _unblacklist(ign):
5789             open(fn, "w").close()
5790             # the Blacklist object watches mtime to tell when the file has
5791             # changed, but on windows this test will run faster than the
5792             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5793             # to force a reload.
5794             self.g.clients[0].blacklist.last_mtime -= 2.0
5795         d.addCallback(_unblacklist)
5796
5797         # now a read should work
5798         d.addCallback(lambda ign: self.GET(self.url))
5799         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5800
5801         # read again to exercise the blacklist-is-unchanged logic
5802         d.addCallback(lambda ign: self.GET(self.url))
5803         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5804
5805         # now add a blacklisted directory, and make sure files under it are
5806         # refused too
5807         def _add_dir(ign):
5808             childnode = c0.create_node_from_uri(self.uri, None)
5809             return c0.create_dirnode({u"child": (childnode,{}) })
5810         d.addCallback(_add_dir)
5811         def _get_dircap(dn):
5812             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5813             self.dir_url_base = "uri/"+dn.get_write_uri()
5814             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5815             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5816             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5817             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5818         d.addCallback(_get_dircap)
5819         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5820         d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5821         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5822         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5823         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5824         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5825         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5826         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5827         d.addCallback(lambda ign: self.GET(self.child_url))
5828         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5829
5830         def _block_dir(ign):
5831             f = open(fn, "w")
5832             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5833             f.close()
5834             self.g.clients[0].blacklist.last_mtime -= 2.0
5835         d.addCallback(_block_dir)
5836         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5837                                                        403, "Forbidden",
5838                                                        "Access Prohibited: dir-off-limits",
5839                                                        self.GET, self.dir_url_base))
5840         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5841                                                        403, "Forbidden",
5842                                                        "Access Prohibited: dir-off-limits",
5843                                                        self.GET, self.dir_url_json1))
5844         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5845                                                        403, "Forbidden",
5846                                                        "Access Prohibited: dir-off-limits",
5847                                                        self.GET, self.dir_url_json2))
5848         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5849                                                        403, "Forbidden",
5850                                                        "Access Prohibited: dir-off-limits",
5851                                                        self.GET, self.dir_url_json_ro))
5852         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5853                                                        403, "Forbidden",
5854                                                        "Access Prohibited: dir-off-limits",
5855                                                        self.GET, self.child_url))
5856         return d
5857
5858
5859 class CompletelyUnhandledError(Exception):
5860     pass
5861 class ErrorBoom(rend.Page):
5862     def beforeRender(self, ctx):
5863         raise CompletelyUnhandledError("whoops")