src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53 DIR_HTML_TAG = '<html lang="en">'
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194     def get_available_space(self):
 195         return 123456
 196
 197 class FakeBucketCounter(object):
 198     def get_state(self):
 199         return {"last-complete-bucket-count": 0}
 200     def get_progress(self):
 201         return {"estimated-time-per-cycle": 0,
 202                 "cycle-in-progress": False,
 203                 "remaining-wait-time": 0}
 204
 205 class FakeLeaseChecker(object):
 206     def __init__(self):
 207         self.expiration_enabled = False
 208         self.mode = "age"
 209         self.override_lease_duration = None
 210         self.sharetypes_to_expire = {}
 211     def get_state(self):
 212         return {"history": None}
 213     def get_progress(self):
 214         return {"estimated-time-per-cycle": 0,
 215                 "cycle-in-progress": False,
 216                 "remaining-wait-time": 0}
 217
 218 class FakeStorageServer(service.MultiService):
 219     name = 'storage'
 220     def __init__(self, nodeid, nickname):
 221         service.MultiService.__init__(self)
 222         self.my_nodeid = nodeid
 223         self.nickname = nickname
 224         self.bucket_counter = FakeBucketCounter()
 225         self.lease_checker = FakeLeaseChecker()
 226     def get_stats(self):
 227         return {"storage_server.accepting_immutable_shares": False}
 228
 229 class FakeClient(Client):
 230     def __init__(self):
 231         # don't upcall to Client.__init__, since we only want to initialize a
 232         # minimal subset
 233         service.MultiService.__init__(self)
 234         self.all_contents = {}
 235         self.nodeid = "fake_nodeid"
 236         self.nickname = u"fake_nickname \u263A"
 237         self.introducer_furl = "None"
 238         self.stats_provider = FakeStatsProvider()
 239         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 240         self.helper = None
 241         self.convergence = "some random string"
 242         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 243         # fake knowledge of another server
 244         self.storage_broker.test_add_server("other_nodeid",
 245                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 246         self.introducer_client = None
 247         self.history = FakeHistory()
 248         self.uploader = FakeUploader()
 249         self.uploader.all_contents = self.all_contents
 250         self.uploader.setServiceParent(self)
 251         self.blacklist = None
 252         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 253                                        self.uploader, None,
 254                                        None, None, None)
 255         self.nodemaker.all_contents = self.all_contents
 256         self.mutable_file_default = SDMF_VERSION
 257         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 258
 259     def get_long_nodeid(self):
 260         return "v0-nodeid"
 261     def get_long_tubid(self):
 262         return "tubid"
 263
 264     def startService(self):
 265         return service.MultiService.startService(self)
 266     def stopService(self):
 267         return service.MultiService.stopService(self)
 268
 269     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 270
 271 class WebMixin(object):
 272     def setUp(self):
 273         self.s = FakeClient()
 274         self.s.startService()
 275         self.staticdir = self.mktemp()
 276         self.clock = Clock()
 277         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 278                                       clock=self.clock)
 279         self.ws.setServiceParent(self.s)
 280         self.webish_port = self.ws.getPortnum()
 281         self.webish_url = self.ws.getURL()
 282         assert self.webish_url.endswith("/")
 283         self.webish_url = self.webish_url[:-1] # these tests add their own /
 284
 285         l = [ self.s.create_dirnode() for x in range(6) ]
 286         d = defer.DeferredList(l)
 287         def _then(res):
 288             self.public_root = res[0][1]
 289             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 290             self.public_url = "/uri/" + self.public_root.get_uri()
 291             self.private_root = res[1][1]
 292
 293             foo = res[2][1]
 294             self._foo_node = foo
 295             self._foo_uri = foo.get_uri()
 296             self._foo_readonly_uri = foo.get_readonly_uri()
 297             self._foo_verifycap = foo.get_verify_cap().to_string()
 298             # NOTE: we ignore the deferred on all set_uri() calls, because we
 299             # know the fake nodes do these synchronously
 300             self.public_root.set_uri(u"foo", foo.get_uri(),
 301                                      foo.get_readonly_uri())
 302
 303             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 304             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 305             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 306
 307             # sdmf
 308             # XXX: Do we ever use this?
 309             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 310
 311             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 312
 313             # mdmf
 314             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 315             assert self._quux_txt_uri.startswith("URI:MDMF")
 316             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 317
 318             foo.set_uri(u"empty", res[3][1].get_uri(),
 319                         res[3][1].get_readonly_uri())
 320             sub_uri = res[4][1].get_uri()
 321             self._sub_uri = sub_uri
 322             foo.set_uri(u"sub", sub_uri, sub_uri)
 323             sub = self.s.create_node_from_uri(sub_uri)
 324             self._sub_node = sub
 325
 326             _ign, n, blocking_uri = self.makefile(1)
 327             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 328
 329             # filenode to test for html encoding issues
 330             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 331             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 332             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 333             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 334             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 335             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 336             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 337             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 338
 339             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 340             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 341             # still think of it as an umlaut
 342             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 343
 344             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 345             self._baz_file_uri = baz_file
 346             sub.set_uri(u"baz.txt", baz_file, baz_file)
 347
 348             _ign, n, self._bad_file_uri = self.makefile(3)
 349             # this uri should not be downloadable
 350             del self.s.all_contents[self._bad_file_uri]
 351
 352             rodir = res[5][1]
 353             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 354                                      rodir.get_readonly_uri())
 355             rodir.set_uri(u"nor", baz_file, baz_file)
 356
 357             # public/
 358             # public/foo/
 359             # public/foo/bar.txt
 360             # public/foo/baz.txt
 361             # public/foo/quux.txt
 362             # public/foo/blockingfile
 363             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 364             # public/foo/empty/
 365             # public/foo/sub/
 366             # public/foo/sub/baz.txt
 367             # public/reedownlee/
 368             # public/reedownlee/nor
 369             self.NEWFILE_CONTENTS = "newfile contents\n"
 370
 371             return foo.get_metadata_for(u"bar.txt")
 372         d.addCallback(_then)
 373         def _got_metadata(metadata):
 374             self._bar_txt_metadata = metadata
 375         d.addCallback(_got_metadata)
 376         return d
 377
 378     def get_all_contents(self):
 379         return self.s.all_contents
 380
 381     def makefile(self, number):
 382         contents = "contents of file %s\n" % number
 383         n = create_chk_filenode(contents, self.get_all_contents())
 384         return contents, n, n.get_uri()
 385
 386     def makefile_mutable(self, number, mdmf=False):
 387         contents = "contents of mutable file %s\n" % number
 388         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 389         return contents, n, n.get_uri(), n.get_readonly_uri()
 390
 391     def tearDown(self):
 392         return self.s.stopService()
 393
 394     def failUnlessIsBarDotTxt(self, res):
 395         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 396
 397     def failUnlessIsQuuxDotTxt(self, res):
 398         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 399
 400     def failUnlessIsBazDotTxt(self, res):
 401         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 402
 403     def failUnlessIsSubBazDotTxt(self, res):
 404         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 405
 406     def failUnlessIsBarJSON(self, res):
 407         data = simplejson.loads(res)
 408         self.failUnless(isinstance(data, list))
 409         self.failUnlessEqual(data[0], "filenode")
 410         self.failUnless(isinstance(data[1], dict))
 411         self.failIf(data[1]["mutable"])
 412         self.failIfIn("rw_uri", data[1]) # immutable
 413         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 414         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 415         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 416
 417     def failUnlessIsQuuxJSON(self, res, readonly=False):
 418         data = simplejson.loads(res)
 419         self.failUnless(isinstance(data, list))
 420         self.failUnlessEqual(data[0], "filenode")
 421         self.failUnless(isinstance(data[1], dict))
 422         metadata = data[1]
 423         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 424
 425     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 426         self.failUnless(metadata['mutable'])
 427         if readonly:
 428             self.failIfIn("rw_uri", metadata)
 429         else:
 430             self.failUnlessIn("rw_uri", metadata)
 431             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 432         self.failUnlessIn("ro_uri", metadata)
 433         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 434         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 435
 436     def failUnlessIsFooJSON(self, res):
 437         data = simplejson.loads(res)
 438         self.failUnless(isinstance(data, list))
 439         self.failUnlessEqual(data[0], "dirnode", res)
 440         self.failUnless(isinstance(data[1], dict))
 441         self.failUnless(data[1]["mutable"])
 442         self.failUnlessIn("rw_uri", data[1]) # mutable
 443         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 444         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 445         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 446
 447         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 448         self.failUnlessEqual(kidnames,
 449                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 450                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 451         kids = dict( [(unicode(name),value)
 452                       for (name,value)
 453                       in data[1]["children"].iteritems()] )
 454         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 455         self.failUnlessIn("metadata", kids[u"sub"][1])
 456         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 457         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 458         self.failUnlessIn("linkcrtime", tahoe_md)
 459         self.failUnlessIn("linkmotime", tahoe_md)
 460         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 461         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 462         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 463         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 464                                    self._bar_txt_verifycap)
 465         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 466         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 467         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 468                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 469         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 470                                    self._bar_txt_uri)
 471         self.failUnlessIn("quux.txt", kids)
 472         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 473                                    self._quux_txt_uri)
 474         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 475                                    self._quux_txt_readonly_uri)
 476
 477     def GET(self, urlpath, followRedirect=False, return_response=False,
 478             **kwargs):
 479         # if return_response=True, this fires with (data, statuscode,
 480         # respheaders) instead of just data.
 481         assert not isinstance(urlpath, unicode)
 482         url = self.webish_url + urlpath
 483         factory = HTTPClientGETFactory(url, method="GET",
 484                                        followRedirect=followRedirect, **kwargs)
 485         reactor.connectTCP("localhost", self.webish_port, factory)
 486         d = factory.deferred
 487         def _got_data(data):
 488             return (data, factory.status, factory.response_headers)
 489         if return_response:
 490             d.addCallback(_got_data)
 491         return factory.deferred
 492
 493     def HEAD(self, urlpath, return_response=False, **kwargs):
 494         # this requires some surgery, because twisted.web.client doesn't want
 495         # to give us back the response headers.
 496         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 497         reactor.connectTCP("localhost", self.webish_port, factory)
 498         d = factory.deferred
 499         def _got_data(data):
 500             return (data, factory.status, factory.response_headers)
 501         if return_response:
 502             d.addCallback(_got_data)
 503         return factory.deferred
 504
 505     def PUT(self, urlpath, data, **kwargs):
 506         url = self.webish_url + urlpath
 507         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 508
 509     def DELETE(self, urlpath):
 510         url = self.webish_url + urlpath
 511         return client.getPage(url, method="DELETE")
 512
 513     def POST(self, urlpath, followRedirect=False, **fields):
 514         sepbase = "boogabooga"
 515         sep = "--" + sepbase
 516         form = []
 517         form.append(sep)
 518         form.append('Content-Disposition: form-data; name="_charset"')
 519         form.append('')
 520         form.append('UTF-8')
 521         form.append(sep)
 522         for name, value in fields.iteritems():
 523             if isinstance(value, tuple):
 524                 filename, value = value
 525                 form.append('Content-Disposition: form-data; name="%s"; '
 526                             'filename="%s"' % (name, filename.encode("utf-8")))
 527             else:
 528                 form.append('Content-Disposition: form-data; name="%s"' % name)
 529             form.append('')
 530             if isinstance(value, unicode):
 531                 value = value.encode("utf-8")
 532             else:
 533                 value = str(value)
 534             assert isinstance(value, str)
 535             form.append(value)
 536             form.append(sep)
 537         form[-1] += "--"
 538         body = ""
 539         headers = {}
 540         if fields:
 541             body = "\r\n".join(form) + "\r\n"
 542             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 543         return self.POST2(urlpath, body, headers, followRedirect)
 544
 545     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 546         url = self.webish_url + urlpath
 547         return client.getPage(url, method="POST", postdata=body,
 548                               headers=headers, followRedirect=followRedirect)
 549
 550     def shouldFail(self, res, expected_failure, which,
 551                    substring=None, response_substring=None):
 552         if isinstance(res, failure.Failure):
 553             res.trap(expected_failure)
 554             if substring:
 555                 self.failUnlessIn(substring, str(res), which)
 556             if response_substring:
 557                 self.failUnlessIn(response_substring, res.value.response, which)
 558         else:
 559             self.fail("%s was supposed to raise %s, not get '%s'" %
 560                       (which, expected_failure, res))
 561
 562     def shouldFail2(self, expected_failure, which, substring,
 563                     response_substring,
 564                     callable, *args, **kwargs):
 565         assert substring is None or isinstance(substring, str)
 566         assert response_substring is None or isinstance(response_substring, str)
 567         d = defer.maybeDeferred(callable, *args, **kwargs)
 568         def done(res):
 569             if isinstance(res, failure.Failure):
 570                 res.trap(expected_failure)
 571                 if substring:
 572                     self.failUnlessIn(substring, str(res),
 573                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 574                                       (substring, str(res),
 575                                        getattr(res.value, "response", ""),
 576                                        which))
 577                 if response_substring:
 578                     self.failUnlessIn(response_substring, res.value.response,
 579                                       "'%s' not in '%s' for test '%s'" % \
 580                                       (response_substring, res.value.response,
 581                                        which))
 582             else:
 583                 self.fail("%s was supposed to raise %s, not get '%s'" %
 584                           (which, expected_failure, res))
 585         d.addBoth(done)
 586         return d
 587
 588     def should404(self, res, which):
 589         if isinstance(res, failure.Failure):
 590             res.trap(error.Error)
 591             self.failUnlessReallyEqual(res.value.status, "404")
 592         else:
 593             self.fail("%s was supposed to Error(404), not get '%s'" %
 594                       (which, res))
 595
 596     def should302(self, res, which):
 597         if isinstance(res, failure.Failure):
 598             res.trap(error.Error)
 599             self.failUnlessReallyEqual(res.value.status, "302")
 600         else:
 601             self.fail("%s was supposed to Error(302), not get '%s'" %
 602                       (which, res))
 603
 604
 605 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 606     def test_create(self):
 607         pass
 608
 609     def test_welcome(self):
 610         d = self.GET("/")
 611         def _check(res):
 612             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 613             self.failUnlessIn(FAVICON_MARKUP, res)
 614             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 615             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 616             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 617             self.failUnlessIn('Page rendered at', res)
 618             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 619             res_u = res.decode('utf-8')
 620             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 621             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 622             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 623             self.failUnlessIn('<td><h3>Available</h3></td>', res)
 624             self.failUnlessIn('123.5kB', res)
 625
 626             self.s.basedir = 'web/test_welcome'
 627             fileutil.make_dirs("web/test_welcome")
 628             fileutil.make_dirs("web/test_welcome/private")
 629             return self.GET("/")
 630         d.addCallback(_check)
 631         return d
 632
 633     def test_introducer_status(self):
 634         class MockIntroducerClient(object):
 635             def __init__(self, connected):
 636                 self.connected = connected
 637             def connected_to_introducer(self):
 638                 return self.connected
 639
 640         d = defer.succeed(None)
 641
 642         # introducer not connected, unguessable furl
 643         def _set_introducer_not_connected_unguessable(ign):
 644             self.s.introducer_furl = "pb://someIntroducer/secret"
 645             self.s.introducer_client = MockIntroducerClient(False)
 646             return self.GET("/")
 647         d.addCallback(_set_introducer_not_connected_unguessable)
 648         def _check_introducer_not_connected_unguessable(res):
 649             html = res.replace('\n', ' ')
 650             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 651             self.failIfIn('pb://someIntroducer/secret', html)
 652             self.failUnless(re.search('<img src="img/connected-no.png" alt="Disconnected" />', html), res)
 653         d.addCallback(_check_introducer_not_connected_unguessable)
 654
 655         # introducer connected, unguessable furl
 656         def _set_introducer_connected_unguessable(ign):
 657             self.s.introducer_furl = "pb://someIntroducer/secret"
 658             self.s.introducer_client = MockIntroducerClient(True)
 659             return self.GET("/")
 660         d.addCallback(_set_introducer_connected_unguessable)
 661         def _check_introducer_connected_unguessable(res):
 662             html = res.replace('\n', ' ')
 663             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 664             self.failIfIn('pb://someIntroducer/secret', html)
 665             self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
 666         d.addCallback(_check_introducer_connected_unguessable)
 667
 668         # introducer connected, guessable furl
 669         def _set_introducer_connected_guessable(ign):
 670             self.s.introducer_furl = "pb://someIntroducer/introducer"
 671             self.s.introducer_client = MockIntroducerClient(True)
 672             return self.GET("/")
 673         d.addCallback(_set_introducer_connected_guessable)
 674         def _check_introducer_connected_guessable(res):
 675             html = res.replace('\n', ' ')
 676             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 677             self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
 678         d.addCallback(_check_introducer_connected_guessable)
 679         return d
 680
 681     def test_helper_status(self):
 682         d = defer.succeed(None)
 683
 684         # set helper furl to None
 685         def _set_no_helper(ign):
 686             self.s.uploader.helper_furl = None
 687             return self.GET("/")
 688         d.addCallback(_set_no_helper)
 689         def _check_no_helper(res):
 690             html = res.replace('\n', ' ')
 691             self.failUnless(re.search('<img src="img/connected-not-configured.png" alt="Not Configured" />', html), res)
 692         d.addCallback(_check_no_helper)
 693
 694         # enable helper, not connected
 695         def _set_helper_not_connected(ign):
 696             self.s.uploader.helper_furl = "pb://someHelper/secret"
 697             self.s.uploader.helper_connected = False
 698             return self.GET("/")
 699         d.addCallback(_set_helper_not_connected)
 700         def _check_helper_not_connected(res):
 701             html = res.replace('\n', ' ')
 702             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 703             self.failIfIn('pb://someHelper/secret', html)
 704             self.failUnless(re.search('<img src="img/connected-no.png" alt="Disconnected" />', html), res)
 705         d.addCallback(_check_helper_not_connected)
 706
 707         # enable helper, connected
 708         def _set_helper_connected(ign):
 709             self.s.uploader.helper_furl = "pb://someHelper/secret"
 710             self.s.uploader.helper_connected = True
 711             return self.GET("/")
 712         d.addCallback(_set_helper_connected)
 713         def _check_helper_connected(res):
 714             html = res.replace('\n', ' ')
 715             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 716             self.failIfIn('pb://someHelper/secret', html)
 717             self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
 718         d.addCallback(_check_helper_connected)
 719         return d
 720
 721     def test_storage(self):
 722         d = self.GET("/storage")
 723         def _check(res):
 724             self.failUnlessIn('Storage Server Status', res)
 725             self.failUnlessIn(FAVICON_MARKUP, res)
 726             res_u = res.decode('utf-8')
 727             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 728         d.addCallback(_check)
 729         return d
 730
 731     def test_status(self):
 732         h = self.s.get_history()
 733         dl_num = h.list_all_download_statuses()[0].get_counter()
 734         ul_num = h.list_all_upload_statuses()[0].get_counter()
 735         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 736         pub_num = h.list_all_publish_statuses()[0].get_counter()
 737         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 738         d = self.GET("/status", followRedirect=True)
 739         def _check(res):
 740             self.failUnlessIn('Recent and Active Operations', res)
 741             self.failUnlessIn('"down-%d"' % dl_num, res)
 742             self.failUnlessIn('"up-%d"' % ul_num, res)
 743             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 744             self.failUnlessIn('"publish-%d"' % pub_num, res)
 745             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 746         d.addCallback(_check)
 747         d.addCallback(lambda res: self.GET("/status/?t=json"))
 748         def _check_json(res):
 749             data = simplejson.loads(res)
 750             self.failUnless(isinstance(data, dict))
 751             #active = data["active"]
 752             # TODO: test more. We need a way to fake an active operation
 753             # here.
 754         d.addCallback(_check_json)
 755
 756         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 757         def _check_dl(res):
 758             self.failUnlessIn("File Download Status", res)
 759         d.addCallback(_check_dl)
 760         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 761         def _check_dl_json(res):
 762             data = simplejson.loads(res)
 763             self.failUnless(isinstance(data, dict))
 764             self.failUnlessIn("read", data)
 765             self.failUnlessEqual(data["read"][0]["length"], 120)
 766             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 767             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 768             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 769             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 770             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 771             # serverids[] keys are strings, since that's what JSON does, but
 772             # we'd really like them to be ints
 773             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 774             self.failUnless(data["serverids"].has_key("1"),
 775                             str(data["serverids"]))
 776             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 777                                  str(data["serverids"]))
 778             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 779                                  "phwrsjte")
 780             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 781                                  "cmpuvkjm")
 782             self.failUnlessIn("dyhb", data)
 783             self.failUnlessIn("misc", data)
 784         d.addCallback(_check_dl_json)
 785         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 786         def _check_ul(res):
 787             self.failUnlessIn("File Upload Status", res)
 788         d.addCallback(_check_ul)
 789         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 790         def _check_mapupdate(res):
 791             self.failUnlessIn("Mutable File Servermap Update Status", res)
 792         d.addCallback(_check_mapupdate)
 793         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 794         def _check_publish(res):
 795             self.failUnlessIn("Mutable File Publish Status", res)
 796         d.addCallback(_check_publish)
 797         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 798         def _check_retrieve(res):
 799             self.failUnlessIn("Mutable File Retrieve Status", res)
 800         d.addCallback(_check_retrieve)
 801
 802         return d
 803
 804     def test_status_numbers(self):
 805         drrm = status.DownloadResultsRendererMixin()
 806         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 807         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 808         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 809         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 810         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 811         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 812         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 813         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 814         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 815
 816         urrm = status.UploadResultsRendererMixin()
 817         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 818         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 819         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 820         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 821         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 822         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 823         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 824         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 825         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 826
 827     def test_GET_FILEURL(self):
 828         d = self.GET(self.public_url + "/foo/bar.txt")
 829         d.addCallback(self.failUnlessIsBarDotTxt)
 830         return d
 831
 832     def test_GET_FILEURL_range(self):
 833         headers = {"range": "bytes=1-10"}
 834         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 835                      return_response=True)
 836         def _got((res, status, headers)):
 837             self.failUnlessReallyEqual(int(status), 206)
 838             self.failUnless(headers.has_key("content-range"))
 839             self.failUnlessReallyEqual(headers["content-range"][0],
 840                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 841             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 842         d.addCallback(_got)
 843         return d
 844
 845     def test_GET_FILEURL_partial_range(self):
 846         headers = {"range": "bytes=5-"}
 847         length  = len(self.BAR_CONTENTS)
 848         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 849                      return_response=True)
 850         def _got((res, status, headers)):
 851             self.failUnlessReallyEqual(int(status), 206)
 852             self.failUnless(headers.has_key("content-range"))
 853             self.failUnlessReallyEqual(headers["content-range"][0],
 854                                        "bytes 5-%d/%d" % (length-1, length))
 855             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 856         d.addCallback(_got)
 857         return d
 858
 859     def test_GET_FILEURL_partial_end_range(self):
 860         headers = {"range": "bytes=-5"}
 861         length  = len(self.BAR_CONTENTS)
 862         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 863                      return_response=True)
 864         def _got((res, status, headers)):
 865             self.failUnlessReallyEqual(int(status), 206)
 866             self.failUnless(headers.has_key("content-range"))
 867             self.failUnlessReallyEqual(headers["content-range"][0],
 868                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 869             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 870         d.addCallback(_got)
 871         return d
 872
 873     def test_GET_FILEURL_partial_range_overrun(self):
 874         headers = {"range": "bytes=100-200"}
 875         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 876                              "416 Requested Range not satisfiable",
 877                              "First beyond end of file",
 878                              self.GET, self.public_url + "/foo/bar.txt",
 879                              headers=headers)
 880         return d
 881
 882     def test_HEAD_FILEURL_range(self):
 883         headers = {"range": "bytes=1-10"}
 884         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 885                      return_response=True)
 886         def _got((res, status, headers)):
 887             self.failUnlessReallyEqual(res, "")
 888             self.failUnlessReallyEqual(int(status), 206)
 889             self.failUnless(headers.has_key("content-range"))
 890             self.failUnlessReallyEqual(headers["content-range"][0],
 891                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 892         d.addCallback(_got)
 893         return d
 894
 895     def test_HEAD_FILEURL_partial_range(self):
 896         headers = {"range": "bytes=5-"}
 897         length  = len(self.BAR_CONTENTS)
 898         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 899                      return_response=True)
 900         def _got((res, status, headers)):
 901             self.failUnlessReallyEqual(int(status), 206)
 902             self.failUnless(headers.has_key("content-range"))
 903             self.failUnlessReallyEqual(headers["content-range"][0],
 904                                        "bytes 5-%d/%d" % (length-1, length))
 905         d.addCallback(_got)
 906         return d
 907
 908     def test_HEAD_FILEURL_partial_end_range(self):
 909         headers = {"range": "bytes=-5"}
 910         length  = len(self.BAR_CONTENTS)
 911         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 912                      return_response=True)
 913         def _got((res, status, headers)):
 914             self.failUnlessReallyEqual(int(status), 206)
 915             self.failUnless(headers.has_key("content-range"))
 916             self.failUnlessReallyEqual(headers["content-range"][0],
 917                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 918         d.addCallback(_got)
 919         return d
 920
 921     def test_HEAD_FILEURL_partial_range_overrun(self):
 922         headers = {"range": "bytes=100-200"}
 923         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 924                              "416 Requested Range not satisfiable",
 925                              "",
 926                              self.HEAD, self.public_url + "/foo/bar.txt",
 927                              headers=headers)
 928         return d
 929
 930     def test_GET_FILEURL_range_bad(self):
 931         headers = {"range": "BOGUS=fizbop-quarnak"}
 932         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 933                      return_response=True)
 934         def _got((res, status, headers)):
 935             self.failUnlessReallyEqual(int(status), 200)
 936             self.failUnless(not headers.has_key("content-range"))
 937             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 938         d.addCallback(_got)
 939         return d
 940
 941     def test_HEAD_FILEURL(self):
 942         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 943         def _got((res, status, headers)):
 944             self.failUnlessReallyEqual(res, "")
 945             self.failUnlessReallyEqual(headers["content-length"][0],
 946                                        str(len(self.BAR_CONTENTS)))
 947             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 948         d.addCallback(_got)
 949         return d
 950
 951     def test_GET_FILEURL_named(self):
 952         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 953         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 954         d = self.GET(base + "/@@name=/blah.txt")
 955         d.addCallback(self.failUnlessIsBarDotTxt)
 956         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 957         d.addCallback(self.failUnlessIsBarDotTxt)
 958         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 959         d.addCallback(self.failUnlessIsBarDotTxt)
 960         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 961         d.addCallback(self.failUnlessIsBarDotTxt)
 962         save_url = base + "?save=true&filename=blah.txt"
 963         d.addCallback(lambda res: self.GET(save_url))
 964         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 965         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 966         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 967         u_url = base + "?save=true&filename=" + u_fn_e
 968         d.addCallback(lambda res: self.GET(u_url))
 969         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 970         return d
 971
 972     def test_PUT_FILEURL_named_bad(self):
 973         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 974         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 975                              "400 Bad Request",
 976                              "/file can only be used with GET or HEAD",
 977                              self.PUT, base + "/@@name=/blah.txt", "")
 978         return d
 979
 980
 981     def test_GET_DIRURL_named_bad(self):
 982         base = "/file/%s" % urllib.quote(self._foo_uri)
 983         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 984                              "400 Bad Request",
 985                              "is not a file-cap",
 986                              self.GET, base + "/@@name=/blah.txt")
 987         return d
 988
 989     def test_GET_slash_file_bad(self):
 990         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 991                              "404 Not Found",
 992                              "/file must be followed by a file-cap and a name",
 993                              self.GET, "/file")
 994         return d
 995
 996     def test_GET_unhandled_URI_named(self):
 997         contents, n, newuri = self.makefile(12)
 998         verifier_cap = n.get_verify_cap().to_string()
 999         base = "/file/%s" % urllib.quote(verifier_cap)
1000         # client.create_node_from_uri() can't handle verify-caps
1001         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1002                              "400 Bad Request", "is not a file-cap",
1003                              self.GET, base)
1004         return d
1005
1006     def test_GET_unhandled_URI(self):
1007         contents, n, newuri = self.makefile(12)
1008         verifier_cap = n.get_verify_cap().to_string()
1009         base = "/uri/%s" % urllib.quote(verifier_cap)
1010         # client.create_node_from_uri() can't handle verify-caps
1011         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1012                              "400 Bad Request",
1013                              "GET unknown URI type: can only do t=info",
1014                              self.GET, base)
1015         return d
1016
1017     def test_GET_FILE_URI(self):
1018         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1019         d = self.GET(base)
1020         d.addCallback(self.failUnlessIsBarDotTxt)
1021         return d
1022
1023     def test_GET_FILE_URI_mdmf(self):
1024         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1025         d = self.GET(base)
1026         d.addCallback(self.failUnlessIsQuuxDotTxt)
1027         return d
1028
1029     def test_GET_FILE_URI_mdmf_extensions(self):
1030         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1031         d = self.GET(base)
1032         d.addCallback(self.failUnlessIsQuuxDotTxt)
1033         return d
1034
1035     def test_GET_FILE_URI_mdmf_readonly(self):
1036         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1037         d = self.GET(base)
1038         d.addCallback(self.failUnlessIsQuuxDotTxt)
1039         return d
1040
1041     def test_GET_FILE_URI_badchild(self):
1042         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1043         errmsg = "Files have no children, certainly not named 'boguschild'"
1044         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1045                              "400 Bad Request", errmsg,
1046                              self.GET, base)
1047         return d
1048
1049     def test_PUT_FILE_URI_badchild(self):
1050         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1051         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1052         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1053                              "400 Bad Request", errmsg,
1054                              self.PUT, base, "")
1055         return d
1056
1057     def test_PUT_FILE_URI_mdmf(self):
1058         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1059         self._quux_new_contents = "new_contents"
1060         d = self.GET(base)
1061         d.addCallback(lambda res:
1062             self.failUnlessIsQuuxDotTxt(res))
1063         d.addCallback(lambda ignored:
1064             self.PUT(base, self._quux_new_contents))
1065         d.addCallback(lambda ignored:
1066             self.GET(base))
1067         d.addCallback(lambda res:
1068             self.failUnlessReallyEqual(res, self._quux_new_contents))
1069         return d
1070
1071     def test_PUT_FILE_URI_mdmf_extensions(self):
1072         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1073         self._quux_new_contents = "new_contents"
1074         d = self.GET(base)
1075         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1076         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1077         d.addCallback(lambda ignored: self.GET(base))
1078         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1079                                                        res))
1080         return d
1081
1082     def test_PUT_FILE_URI_mdmf_readonly(self):
1083         # We're not allowed to PUT things to a readonly cap.
1084         base = "/uri/%s" % self._quux_txt_readonly_uri
1085         d = self.GET(base)
1086         d.addCallback(lambda res:
1087             self.failUnlessIsQuuxDotTxt(res))
1088         # What should we get here? We get a 500 error now; that's not right.
1089         d.addCallback(lambda ignored:
1090             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1091                              "400 Bad Request", "read-only cap",
1092                              self.PUT, base, "new data"))
1093         return d
1094
1095     def test_PUT_FILE_URI_sdmf_readonly(self):
1096         # We're not allowed to put things to a readonly cap.
1097         base = "/uri/%s" % self._baz_txt_readonly_uri
1098         d = self.GET(base)
1099         d.addCallback(lambda res:
1100             self.failUnlessIsBazDotTxt(res))
1101         d.addCallback(lambda ignored:
1102             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1103                              "400 Bad Request", "read-only cap",
1104                              self.PUT, base, "new_data"))
1105         return d
1106
1107     def test_GET_etags(self):
1108
1109         def _check_etags(uri):
1110             d1 = _get_etag(uri)
1111             d2 = _get_etag(uri, 'json')
1112             d = defer.DeferredList([d1, d2], consumeErrors=True)
1113             def _check(results):
1114                 # All deferred must succeed
1115                 self.failUnless(all([r[0] for r in results]))
1116                 # the etag for the t=json form should be just like the etag
1117                 # fo the default t='' form, but with a 'json' suffix
1118                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1119             d.addCallback(_check)
1120             return d
1121
1122         def _get_etag(uri, t=''):
1123             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1124             d = self.GET(targetbase, return_response=True, followRedirect=True)
1125             def _just_the_etag(result):
1126                 data, response, headers = result
1127                 etag = headers['etag'][0]
1128                 if uri.startswith('URI:DIR'):
1129                     self.failUnless(etag.startswith('DIR:'), etag)
1130                 return etag
1131             return d.addCallback(_just_the_etag)
1132
1133         # Check that etags work with immutable directories
1134         (newkids, caps) = self._create_immutable_children()
1135         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1136                       simplejson.dumps(newkids))
1137         def _stash_immdir_uri(uri):
1138             self._immdir_uri = uri
1139             return uri
1140         d.addCallback(_stash_immdir_uri)
1141         d.addCallback(_check_etags)
1142
1143         # Check that etags work with immutable files
1144         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1145
1146         # use the ETag on GET
1147         def _check_match(ign):
1148             uri = "/uri/%s" % self._bar_txt_uri
1149             d = self.GET(uri, return_response=True)
1150             # extract the ETag
1151             d.addCallback(lambda (data, code, headers):
1152                           headers['etag'][0])
1153             # do a GET that's supposed to match the ETag
1154             d.addCallback(lambda etag:
1155                           self.GET(uri, return_response=True,
1156                                    headers={"If-None-Match": etag}))
1157             # make sure it short-circuited (304 instead of 200)
1158             d.addCallback(lambda (data, code, headers):
1159                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1160             return d
1161         d.addCallback(_check_match)
1162
1163         def _no_etag(uri, t):
1164             target = "/uri/%s?t=%s" % (uri, t)
1165             d = self.GET(target, return_response=True, followRedirect=True)
1166             d.addCallback(lambda (data, code, headers):
1167                           self.failIf("etag" in headers, target))
1168             return d
1169         def _yes_etag(uri, t):
1170             target = "/uri/%s?t=%s" % (uri, t)
1171             d = self.GET(target, return_response=True, followRedirect=True)
1172             d.addCallback(lambda (data, code, headers):
1173                           self.failUnless("etag" in headers, target))
1174             return d
1175
1176         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1177         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1178         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1179         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1180         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1181
1182         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1183         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1184         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1185         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1186         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1187         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1188
1189         return d
1190
1191     # TODO: version of this with a Unicode filename
1192     def test_GET_FILEURL_save(self):
1193         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1194                      return_response=True)
1195         def _got((res, statuscode, headers)):
1196             content_disposition = headers["content-disposition"][0]
1197             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1198             self.failUnlessIsBarDotTxt(res)
1199         d.addCallback(_got)
1200         return d
1201
1202     def test_GET_FILEURL_missing(self):
1203         d = self.GET(self.public_url + "/foo/missing")
1204         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1205         return d
1206
1207     def test_GET_FILEURL_info_mdmf(self):
1208         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1209         def _got(res):
1210             self.failUnlessIn("mutable file (mdmf)", res)
1211             self.failUnlessIn(self._quux_txt_uri, res)
1212             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1213         d.addCallback(_got)
1214         return d
1215
1216     def test_GET_FILEURL_info_mdmf_readonly(self):
1217         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1218         def _got(res):
1219             self.failUnlessIn("mutable file (mdmf)", res)
1220             self.failIfIn(self._quux_txt_uri, res)
1221             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1222         d.addCallback(_got)
1223         return d
1224
1225     def test_GET_FILEURL_info_sdmf(self):
1226         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1227         def _got(res):
1228             self.failUnlessIn("mutable file (sdmf)", res)
1229             self.failUnlessIn(self._baz_txt_uri, res)
1230         d.addCallback(_got)
1231         return d
1232
1233     def test_GET_FILEURL_info_mdmf_extensions(self):
1234         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1235         def _got(res):
1236             self.failUnlessIn("mutable file (mdmf)", res)
1237             self.failUnlessIn(self._quux_txt_uri, res)
1238             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1239         d.addCallback(_got)
1240         return d
1241
1242     def test_PUT_overwrite_only_files(self):
1243         # create a directory, put a file in that directory.
1244         contents, n, filecap = self.makefile(8)
1245         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1246         d.addCallback(lambda res:
1247             self.PUT(self.public_url + "/foo/dir/file1.txt",
1248                      self.NEWFILE_CONTENTS))
1249         # try to overwrite the file with replace=only-files
1250         # (this should work)
1251         d.addCallback(lambda res:
1252             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1253                      filecap))
1254         d.addCallback(lambda res:
1255             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1256                  "There was already a child by that name, and you asked me "
1257                  "to not replace it",
1258                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1259                  filecap))
1260         return d
1261
1262     def test_PUT_NEWFILEURL(self):
1263         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1264         # TODO: we lose the response code, so we can't check this
1265         #self.failUnlessReallyEqual(responsecode, 201)
1266         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1267         d.addCallback(lambda res:
1268                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1269                                                       self.NEWFILE_CONTENTS))
1270         return d
1271
1272     def test_PUT_NEWFILEURL_not_mutable(self):
1273         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1274                      self.NEWFILE_CONTENTS)
1275         # TODO: we lose the response code, so we can't check this
1276         #self.failUnlessReallyEqual(responsecode, 201)
1277         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1278         d.addCallback(lambda res:
1279                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1280                                                       self.NEWFILE_CONTENTS))
1281         return d
1282
1283     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1284         # this should get us a few segments of an MDMF mutable file,
1285         # which we can then test for.
1286         contents = self.NEWFILE_CONTENTS * 300000
1287         d = self.PUT("/uri?format=mdmf",
1288                      contents)
1289         def _got_filecap(filecap):
1290             self.failUnless(filecap.startswith("URI:MDMF"))
1291             return filecap
1292         d.addCallback(_got_filecap)
1293         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1294         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1295         return d
1296
1297     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1298         contents = self.NEWFILE_CONTENTS * 300000
1299         d = self.PUT("/uri?format=sdmf",
1300                      contents)
1301         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1302         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1303         return d
1304
1305     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1306         contents = self.NEWFILE_CONTENTS * 300000
1307         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1308                                     400, "Bad Request", "Unknown format: foo",
1309                                     self.PUT, "/uri?format=foo",
1310                                     contents)
1311
1312     def test_PUT_NEWFILEURL_range_bad(self):
1313         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1314         target = self.public_url + "/foo/new.txt"
1315         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1316                              "501 Not Implemented",
1317                              "Content-Range in PUT not yet supported",
1318                              # (and certainly not for immutable files)
1319                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1320                              headers=headers)
1321         d.addCallback(lambda res:
1322                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1323         return d
1324
1325     def test_PUT_NEWFILEURL_mutable(self):
1326         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1327                      self.NEWFILE_CONTENTS)
1328         # TODO: we lose the response code, so we can't check this
1329         #self.failUnlessReallyEqual(responsecode, 201)
1330         def _check_uri(res):
1331             u = uri.from_string_mutable_filenode(res)
1332             self.failUnless(u.is_mutable())
1333             self.failIf(u.is_readonly())
1334             return res
1335         d.addCallback(_check_uri)
1336         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1337         d.addCallback(lambda res:
1338                       self.failUnlessMutableChildContentsAre(self._foo_node,
1339                                                              u"new.txt",
1340                                                              self.NEWFILE_CONTENTS))
1341         return d
1342
1343     def test_PUT_NEWFILEURL_mutable_toobig(self):
1344         # It is okay to upload large mutable files, so we should be able
1345         # to do that.
1346         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1347                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1348         return d
1349
1350     def test_PUT_NEWFILEURL_replace(self):
1351         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1352         # TODO: we lose the response code, so we can't check this
1353         #self.failUnlessReallyEqual(responsecode, 200)
1354         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1355         d.addCallback(lambda res:
1356                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1357                                                       self.NEWFILE_CONTENTS))
1358         return d
1359
1360     def test_PUT_NEWFILEURL_bad_t(self):
1361         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1362                              "PUT to a file: bad t=bogus",
1363                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1364                              "contents")
1365         return d
1366
1367     def test_PUT_NEWFILEURL_no_replace(self):
1368         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1369                      self.NEWFILE_CONTENTS)
1370         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1371                   "409 Conflict",
1372                   "There was already a child by that name, and you asked me "
1373                   "to not replace it")
1374         return d
1375
1376     def test_PUT_NEWFILEURL_mkdirs(self):
1377         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1378         fn = self._foo_node
1379         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1380         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1381         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1382         d.addCallback(lambda res:
1383                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1384                                                       self.NEWFILE_CONTENTS))
1385         return d
1386
1387     def test_PUT_NEWFILEURL_blocked(self):
1388         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1389                      self.NEWFILE_CONTENTS)
1390         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1391                   "409 Conflict",
1392                   "Unable to create directory 'blockingfile': a file was in the way")
1393         return d
1394
1395     def test_PUT_NEWFILEURL_emptyname(self):
1396         # an empty pathname component (i.e. a double-slash) is disallowed
1397         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1398                              "400 Bad Request",
1399                              "The webapi does not allow empty pathname components",
1400                              self.PUT, self.public_url + "/foo//new.txt", "")
1401         return d
1402
1403     def test_DELETE_FILEURL(self):
1404         d = self.DELETE(self.public_url + "/foo/bar.txt")
1405         d.addCallback(lambda res:
1406                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1407         return d
1408
1409     def test_DELETE_FILEURL_missing(self):
1410         d = self.DELETE(self.public_url + "/foo/missing")
1411         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1412         return d
1413
1414     def test_DELETE_FILEURL_missing2(self):
1415         d = self.DELETE(self.public_url + "/missing/missing")
1416         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1417         return d
1418
1419     def failUnlessHasBarDotTxtMetadata(self, res):
1420         data = simplejson.loads(res)
1421         self.failUnless(isinstance(data, list))
1422         self.failUnlessIn("metadata", data[1])
1423         self.failUnlessIn("tahoe", data[1]["metadata"])
1424         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1425         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1426         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1427                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1428
1429     def test_GET_FILEURL_json(self):
1430         # twisted.web.http.parse_qs ignores any query args without an '=', so
1431         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1432         # instead. This may make it tricky to emulate the S3 interface
1433         # completely.
1434         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1435         def _check1(data):
1436             self.failUnlessIsBarJSON(data)
1437             self.failUnlessHasBarDotTxtMetadata(data)
1438             return
1439         d.addCallback(_check1)
1440         return d
1441
1442     def test_GET_FILEURL_json_mutable_type(self):
1443         # The JSON should include format, which says whether the
1444         # file is SDMF or MDMF
1445         d = self.PUT("/uri?format=mdmf",
1446                      self.NEWFILE_CONTENTS * 300000)
1447         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1448         def _got_json(json, version):
1449             data = simplejson.loads(json)
1450             assert "filenode" == data[0]
1451             data = data[1]
1452             assert isinstance(data, dict)
1453
1454             self.failUnlessIn("format", data)
1455             self.failUnlessEqual(data["format"], version)
1456
1457         d.addCallback(_got_json, "MDMF")
1458         # Now make an SDMF file and check that it is reported correctly.
1459         d.addCallback(lambda ignored:
1460             self.PUT("/uri?format=sdmf",
1461                       self.NEWFILE_CONTENTS * 300000))
1462         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1463         d.addCallback(_got_json, "SDMF")
1464         return d
1465
1466     def test_GET_FILEURL_json_mdmf(self):
1467         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1468         d.addCallback(self.failUnlessIsQuuxJSON)
1469         return d
1470
1471     def test_GET_FILEURL_json_missing(self):
1472         d = self.GET(self.public_url + "/foo/missing?json")
1473         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1474         return d
1475
1476     def test_GET_FILEURL_uri(self):
1477         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1478         def _check(res):
1479             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1480         d.addCallback(_check)
1481         d.addCallback(lambda res:
1482                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1483         def _check2(res):
1484             # for now, for files, uris and readonly-uris are the same
1485             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1486         d.addCallback(_check2)
1487         return d
1488
1489     def test_GET_FILEURL_badtype(self):
1490         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1491                                  "bad t=bogus",
1492                                  self.GET,
1493                                  self.public_url + "/foo/bar.txt?t=bogus")
1494         return d
1495
1496     def test_CSS_FILE(self):
1497         d = self.GET("/tahoe.css", followRedirect=True)
1498         def _check(res):
1499             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1500             self.failUnless(CSS_STYLE.search(res), res)
1501         d.addCallback(_check)
1502         return d
1503
1504     def test_GET_FILEURL_uri_missing(self):
1505         d = self.GET(self.public_url + "/foo/missing?t=uri")
1506         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1507         return d
1508
1509     def _check_upload_and_mkdir_forms(self, html):
1510         # We should have a form to create a file, with radio buttons that allow
1511         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1512         self.failUnlessIn('name="t" value="upload"', html)
1513         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1514         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1515         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1516
1517         # We should also have the ability to create a mutable directory, with
1518         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1519         # or MDMF directory.
1520         self.failUnlessIn('name="t" value="mkdir"', html)
1521         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1522         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1523
1524         self.failUnlessIn(FAVICON_MARKUP, html)
1525
1526     def test_GET_DIRECTORY_html(self):
1527         d = self.GET(self.public_url + "/foo", followRedirect=True)
1528         def _check(html):
1529             self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1530             self._check_upload_and_mkdir_forms(html)
1531             self.failUnlessIn("quux", html)
1532         d.addCallback(_check)
1533         return d
1534
1535     def test_GET_DIRECTORY_html_filenode_encoding(self):
1536         d = self.GET(self.public_url + "/foo", followRedirect=True)
1537         def _check(html):
1538             # Check if encoded entries are there
1539             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1540                               + self._htmlname_escaped + '</a>', html)
1541             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1542             self.failIfIn(self._htmlname_escaped_double, html)
1543             # Make sure that Nevow escaping actually works by checking for unsafe characters
1544             # and that '&' is escaped.
1545             for entity in '<>':
1546                 self.failUnlessIn(entity, self._htmlname_raw)
1547                 self.failIfIn(entity, self._htmlname_escaped)
1548             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1549             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1550         d.addCallback(_check)
1551         return d
1552
1553     def test_GET_root_html(self):
1554         d = self.GET("/")
1555         d.addCallback(self._check_upload_and_mkdir_forms)
1556         return d
1557
1558     def test_GET_DIRURL(self):
1559         # the addSlash means we get a redirect here
1560         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1561         ROOT = "../../.."
1562         d = self.GET(self.public_url + "/foo", followRedirect=True)
1563         def _check(res):
1564             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1565
1566             # the FILE reference points to a URI, but it should end in bar.txt
1567             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1568                        (ROOT, urllib.quote(self._bar_txt_uri)))
1569             get_bar = "".join([r'<td>FILE</td>',
1570                                r'\s+<td>',
1571                                r'<a href="%s">bar.txt</a>' % bar_url,
1572                                r'</td>',
1573                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1574                                ])
1575             self.failUnless(re.search(get_bar, res), res)
1576             for label in ['unlink', 'rename/relink']:
1577                 for line in res.split("\n"):
1578                     # find the line that contains the relevant button for bar.txt
1579                     if ("form action" in line and
1580                         ('value="%s"' % (label,)) in line and
1581                         'value="bar.txt"' in line):
1582                         # the form target should use a relative URL
1583                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1584                         self.failUnlessIn('action="%s"' % foo_url, line)
1585                         # and the when_done= should too
1586                         #done_url = urllib.quote(???)
1587                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1588
1589                         # 'unlink' needs to use POST because it directly has a side effect
1590                         if label == 'unlink':
1591                             self.failUnlessIn('method="post"', line)
1592                         break
1593                 else:
1594                     self.fail("unable to find '%s bar.txt' line" % (label,))
1595
1596             # the DIR reference just points to a URI
1597             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1598             get_sub = ((r'<td>DIR</td>')
1599                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1600             self.failUnless(re.search(get_sub, res), res)
1601         d.addCallback(_check)
1602
1603         # look at a readonly directory
1604         d.addCallback(lambda res:
1605                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1606         def _check2(res):
1607             self.failUnlessIn("(read-only)", res)
1608             self.failIfIn("Upload a file", res)
1609         d.addCallback(_check2)
1610
1611         # and at a directory that contains a readonly directory
1612         d.addCallback(lambda res:
1613                       self.GET(self.public_url, followRedirect=True))
1614         def _check3(res):
1615             self.failUnless(re.search('<td>DIR-RO</td>'
1616                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1617         d.addCallback(_check3)
1618
1619         # and an empty directory
1620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1621         def _check4(res):
1622             self.failUnlessIn("directory is empty", res)
1623             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" class="btn" value="Create" />', re.I)
1624             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1625         d.addCallback(_check4)
1626
1627         # and at a literal directory
1628         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1629         d.addCallback(lambda res:
1630                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1631         def _check5(res):
1632             self.failUnlessIn('(immutable)', res)
1633             self.failUnless(re.search('<td>FILE</td>'
1634                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1635         d.addCallback(_check5)
1636         return d
1637
1638     def test_GET_DIRURL_badtype(self):
1639         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1640                                  400, "Bad Request",
1641                                  "bad t=bogus",
1642                                  self.GET,
1643                                  self.public_url + "/foo?t=bogus")
1644         return d
1645
1646     def test_GET_DIRURL_json(self):
1647         d = self.GET(self.public_url + "/foo?t=json")
1648         d.addCallback(self.failUnlessIsFooJSON)
1649         return d
1650
1651     def test_GET_DIRURL_json_format(self):
1652         d = self.PUT(self.public_url + \
1653                      "/foo/sdmf.txt?format=sdmf",
1654                      self.NEWFILE_CONTENTS * 300000)
1655         d.addCallback(lambda ignored:
1656             self.PUT(self.public_url + \
1657                      "/foo/mdmf.txt?format=mdmf",
1658                      self.NEWFILE_CONTENTS * 300000))
1659         # Now we have an MDMF and SDMF file in the directory. If we GET
1660         # its JSON, we should see their encodings.
1661         d.addCallback(lambda ignored:
1662             self.GET(self.public_url + "/foo?t=json"))
1663         def _got_json(json):
1664             data = simplejson.loads(json)
1665             assert data[0] == "dirnode"
1666
1667             data = data[1]
1668             kids = data['children']
1669
1670             mdmf_data = kids['mdmf.txt'][1]
1671             self.failUnlessIn("format", mdmf_data)
1672             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1673
1674             sdmf_data = kids['sdmf.txt'][1]
1675             self.failUnlessIn("format", sdmf_data)
1676             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1677         d.addCallback(_got_json)
1678         return d
1679
1680
1681     def test_POST_DIRURL_manifest_no_ophandle(self):
1682         d = self.shouldFail2(error.Error,
1683                              "test_POST_DIRURL_manifest_no_ophandle",
1684                              "400 Bad Request",
1685                              "slow operation requires ophandle=",
1686                              self.POST, self.public_url, t="start-manifest")
1687         return d
1688
1689     def test_POST_DIRURL_manifest(self):
1690         d = defer.succeed(None)
1691         def getman(ignored, output):
1692             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1693                           followRedirect=True)
1694             d.addCallback(self.wait_for_operation, "125")
1695             d.addCallback(self.get_operation_results, "125", output)
1696             return d
1697         d.addCallback(getman, None)
1698         def _got_html(manifest):
1699             self.failUnlessIn("Manifest of SI=", manifest)
1700             self.failUnlessIn("<td>sub</td>", manifest)
1701             self.failUnlessIn(self._sub_uri, manifest)
1702             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1703             self.failUnlessIn(FAVICON_MARKUP, manifest)
1704         d.addCallback(_got_html)
1705
1706         # both t=status and unadorned GET should be identical
1707         d.addCallback(lambda res: self.GET("/operations/125"))
1708         d.addCallback(_got_html)
1709
1710         d.addCallback(getman, "html")
1711         d.addCallback(_got_html)
1712         d.addCallback(getman, "text")
1713         def _got_text(manifest):
1714             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1715             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1716         d.addCallback(_got_text)
1717         d.addCallback(getman, "JSON")
1718         def _got_json(res):
1719             data = res["manifest"]
1720             got = {}
1721             for (path_list, cap) in data:
1722                 got[tuple(path_list)] = cap
1723             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1724             self.failUnlessIn((u"sub", u"baz.txt"), got)
1725             self.failUnlessIn("finished", res)
1726             self.failUnlessIn("origin", res)
1727             self.failUnlessIn("storage-index", res)
1728             self.failUnlessIn("verifycaps", res)
1729             self.failUnlessIn("stats", res)
1730         d.addCallback(_got_json)
1731         return d
1732
1733     def test_POST_DIRURL_deepsize_no_ophandle(self):
1734         d = self.shouldFail2(error.Error,
1735                              "test_POST_DIRURL_deepsize_no_ophandle",
1736                              "400 Bad Request",
1737                              "slow operation requires ophandle=",
1738                              self.POST, self.public_url, t="start-deep-size")
1739         return d
1740
1741     def test_POST_DIRURL_deepsize(self):
1742         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1743                       followRedirect=True)
1744         d.addCallback(self.wait_for_operation, "126")
1745         d.addCallback(self.get_operation_results, "126", "json")
1746         def _got_json(data):
1747             self.failUnlessReallyEqual(data["finished"], True)
1748             size = data["size"]
1749             self.failUnless(size > 1000)
1750         d.addCallback(_got_json)
1751         d.addCallback(self.get_operation_results, "126", "text")
1752         def _got_text(res):
1753             mo = re.search(r'^size: (\d+)$', res, re.M)
1754             self.failUnless(mo, res)
1755             size = int(mo.group(1))
1756             # with directories, the size varies.
1757             self.failUnless(size > 1000)
1758         d.addCallback(_got_text)
1759         return d
1760
1761     def test_POST_DIRURL_deepstats_no_ophandle(self):
1762         d = self.shouldFail2(error.Error,
1763                              "test_POST_DIRURL_deepstats_no_ophandle",
1764                              "400 Bad Request",
1765                              "slow operation requires ophandle=",
1766                              self.POST, self.public_url, t="start-deep-stats")
1767         return d
1768
1769     def test_POST_DIRURL_deepstats(self):
1770         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1771                       followRedirect=True)
1772         d.addCallback(self.wait_for_operation, "127")
1773         d.addCallback(self.get_operation_results, "127", "json")
1774         def _got_json(stats):
1775             expected = {"count-immutable-files": 4,
1776                         "count-mutable-files": 2,
1777                         "count-literal-files": 0,
1778                         "count-files": 6,
1779                         "count-directories": 3,
1780                         "size-immutable-files": 76,
1781                         "size-literal-files": 0,
1782                         #"size-directories": 1912, # varies
1783                         #"largest-directory": 1590,
1784                         "largest-directory-children": 8,
1785                         "largest-immutable-file": 19,
1786                         }
1787             for k,v in expected.iteritems():
1788                 self.failUnlessReallyEqual(stats[k], v,
1789                                            "stats[%s] was %s, not %s" %
1790                                            (k, stats[k], v))
1791             self.failUnlessReallyEqual(stats["size-files-histogram"],
1792                                        [ [11, 31, 4] ])
1793         d.addCallback(_got_json)
1794         return d
1795
1796     def test_POST_DIRURL_stream_manifest(self):
1797         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1798         def _check(res):
1799             self.failUnless(res.endswith("\n"))
1800             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1801             self.failUnlessReallyEqual(len(units), 10)
1802             self.failUnlessEqual(units[-1]["type"], "stats")
1803             first = units[0]
1804             self.failUnlessEqual(first["path"], [])
1805             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1806             self.failUnlessEqual(first["type"], "directory")
1807             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1808             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1809             self.failIfEqual(baz["storage-index"], None)
1810             self.failIfEqual(baz["verifycap"], None)
1811             self.failIfEqual(baz["repaircap"], None)
1812             # XXX: Add quux and baz to this test.
1813             return
1814         d.addCallback(_check)
1815         return d
1816
1817     def test_GET_DIRURL_uri(self):
1818         d = self.GET(self.public_url + "/foo?t=uri")
1819         def _check(res):
1820             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1821         d.addCallback(_check)
1822         return d
1823
1824     def test_GET_DIRURL_readonly_uri(self):
1825         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1826         def _check(res):
1827             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1828         d.addCallback(_check)
1829         return d
1830
1831     def test_PUT_NEWDIRURL(self):
1832         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1833         d.addCallback(lambda res:
1834                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1835         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1836         d.addCallback(self.failUnlessNodeKeysAre, [])
1837         return d
1838
1839     def test_PUT_NEWDIRURL_mdmf(self):
1840         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1841         d.addCallback(lambda res:
1842                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1843         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1844         d.addCallback(lambda node:
1845             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1846         return d
1847
1848     def test_PUT_NEWDIRURL_sdmf(self):
1849         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1850                      "")
1851         d.addCallback(lambda res:
1852                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1853         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1854         d.addCallback(lambda node:
1855             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1856         return d
1857
1858     def test_PUT_NEWDIRURL_bad_format(self):
1859         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1860                                     400, "Bad Request", "Unknown format: foo",
1861                                     self.PUT, self.public_url +
1862                                     "/foo/newdir=?t=mkdir&format=foo", "")
1863
1864     def test_POST_NEWDIRURL(self):
1865         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1866         d.addCallback(lambda res:
1867                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1868         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1869         d.addCallback(self.failUnlessNodeKeysAre, [])
1870         return d
1871
1872     def test_POST_NEWDIRURL_mdmf(self):
1873         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1874         d.addCallback(lambda res:
1875                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1876         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1877         d.addCallback(lambda node:
1878             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1879         return d
1880
1881     def test_POST_NEWDIRURL_sdmf(self):
1882         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1883         d.addCallback(lambda res:
1884             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1885         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1886         d.addCallback(lambda node:
1887             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1888         return d
1889
1890     def test_POST_NEWDIRURL_bad_format(self):
1891         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1892                                     400, "Bad Request", "Unknown format: foo",
1893                                     self.POST2, self.public_url + \
1894                                     "/foo/newdir?t=mkdir&format=foo", "")
1895
1896     def test_POST_NEWDIRURL_emptyname(self):
1897         # an empty pathname component (i.e. a double-slash) is disallowed
1898         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1899                              "400 Bad Request",
1900                              "The webapi does not allow empty pathname components, i.e. a double slash",
1901                              self.POST, self.public_url + "//?t=mkdir")
1902         return d
1903
1904     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1905         (newkids, caps) = self._create_initial_children()
1906         query = "/foo/newdir?t=mkdir-with-children"
1907         if version == MDMF_VERSION:
1908             query += "&format=mdmf"
1909         elif version == SDMF_VERSION:
1910             query += "&format=sdmf"
1911         else:
1912             version = SDMF_VERSION # for later
1913         d = self.POST2(self.public_url + query,
1914                        simplejson.dumps(newkids))
1915         def _check(uri):
1916             n = self.s.create_node_from_uri(uri.strip())
1917             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1918             self.failUnlessEqual(n._node.get_version(), version)
1919             d2.addCallback(lambda ign:
1920                            self.failUnlessROChildURIIs(n, u"child-imm",
1921                                                        caps['filecap1']))
1922             d2.addCallback(lambda ign:
1923                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1924                                                        caps['filecap2']))
1925             d2.addCallback(lambda ign:
1926                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1927                                                        caps['filecap3']))
1928             d2.addCallback(lambda ign:
1929                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1930                                                        caps['unknown_rocap']))
1931             d2.addCallback(lambda ign:
1932                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1933                                                        caps['unknown_rwcap']))
1934             d2.addCallback(lambda ign:
1935                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1936                                                        caps['unknown_immcap']))
1937             d2.addCallback(lambda ign:
1938                            self.failUnlessRWChildURIIs(n, u"dirchild",
1939                                                        caps['dircap']))
1940             d2.addCallback(lambda ign:
1941                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1942                                                        caps['litdircap']))
1943             d2.addCallback(lambda ign:
1944                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1945                                                        caps['emptydircap']))
1946             return d2
1947         d.addCallback(_check)
1948         d.addCallback(lambda res:
1949                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1950         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1951         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1952         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1953         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1954         return d
1955
1956     def test_POST_NEWDIRURL_initial_children(self):
1957         return self._do_POST_NEWDIRURL_initial_children_test()
1958
1959     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1960         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1961
1962     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1963         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1964
1965     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1966         (newkids, caps) = self._create_initial_children()
1967         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1968                                     400, "Bad Request", "Unknown format: foo",
1969                                     self.POST2, self.public_url + \
1970                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1971                                     simplejson.dumps(newkids))
1972
1973     def test_POST_NEWDIRURL_immutable(self):
1974         (newkids, caps) = self._create_immutable_children()
1975         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1976                        simplejson.dumps(newkids))
1977         def _check(uri):
1978             n = self.s.create_node_from_uri(uri.strip())
1979             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1980             d2.addCallback(lambda ign:
1981                            self.failUnlessROChildURIIs(n, u"child-imm",
1982                                                        caps['filecap1']))
1983             d2.addCallback(lambda ign:
1984                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1985                                                        caps['unknown_immcap']))
1986             d2.addCallback(lambda ign:
1987                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1988                                                        caps['immdircap']))
1989             d2.addCallback(lambda ign:
1990                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1991                                                        caps['litdircap']))
1992             d2.addCallback(lambda ign:
1993                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1994                                                        caps['emptydircap']))
1995             return d2
1996         d.addCallback(_check)
1997         d.addCallback(lambda res:
1998                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2007         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2008         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2009         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2010         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2011         d.addErrback(self.explain_web_error)
2012         return d
2013
2014     def test_POST_NEWDIRURL_immutable_bad(self):
2015         (newkids, caps) = self._create_initial_children()
2016         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2017                              "400 Bad Request",
2018                              "needed to be immutable but was not",
2019                              self.POST2,
2020                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2021                              simplejson.dumps(newkids))
2022         return d
2023
2024     def test_PUT_NEWDIRURL_exists(self):
2025         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2026         d.addCallback(lambda res:
2027                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2028         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2029         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2030         return d
2031
2032     def test_PUT_NEWDIRURL_blocked(self):
2033         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2034                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2035                              self.PUT,
2036                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2037         d.addCallback(lambda res:
2038                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2039         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2040         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2041         return d
2042
2043     def test_PUT_NEWDIRURL_mkdirs(self):
2044         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2045         d.addCallback(lambda res:
2046                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2047         d.addCallback(lambda res:
2048                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2049         d.addCallback(lambda res:
2050                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2051         d.addCallback(self.failUnlessNodeKeysAre, [])
2052         return d
2053
2054     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2055         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2056         d.addCallback(lambda ignored:
2057             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2058         d.addCallback(lambda ignored:
2059             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2060         d.addCallback(lambda ignored:
2061             self._foo_node.get_child_at_path(u"subdir"))
2062         def _got_subdir(subdir):
2063             # XXX: What we want?
2064             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2065             self.failUnlessNodeHasChild(subdir, u"newdir")
2066             return subdir.get_child_at_path(u"newdir")
2067         d.addCallback(_got_subdir)
2068         d.addCallback(lambda newdir:
2069             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2070         return d
2071
2072     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2073         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2074         d.addCallback(lambda ignored:
2075             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2076         d.addCallback(lambda ignored:
2077             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2078         d.addCallback(lambda ignored:
2079             self._foo_node.get_child_at_path(u"subdir"))
2080         def _got_subdir(subdir):
2081             # XXX: What we want?
2082             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2083             self.failUnlessNodeHasChild(subdir, u"newdir")
2084             return subdir.get_child_at_path(u"newdir")
2085         d.addCallback(_got_subdir)
2086         d.addCallback(lambda newdir:
2087             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2088         return d
2089
2090     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2091         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2092                                     400, "Bad Request", "Unknown format: foo",
2093                                     self.PUT, self.public_url + \
2094                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2095                                     "")
2096
2097     def test_DELETE_DIRURL(self):
2098         d = self.DELETE(self.public_url + "/foo")
2099         d.addCallback(lambda res:
2100                       self.failIfNodeHasChild(self.public_root, u"foo"))
2101         return d
2102
2103     def test_DELETE_DIRURL_missing(self):
2104         d = self.DELETE(self.public_url + "/foo/missing")
2105         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2106         d.addCallback(lambda res:
2107                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2108         return d
2109
2110     def test_DELETE_DIRURL_missing2(self):
2111         d = self.DELETE(self.public_url + "/missing")
2112         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2113         return d
2114
2115     def dump_root(self):
2116         print "NODEWALK"
2117         w = webish.DirnodeWalkerMixin()
2118         def visitor(childpath, childnode, metadata):
2119             print childpath
2120         d = w.walk(self.public_root, visitor)
2121         return d
2122
2123     def failUnlessNodeKeysAre(self, node, expected_keys):
2124         for k in expected_keys:
2125             assert isinstance(k, unicode)
2126         d = node.list()
2127         def _check(children):
2128             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2129         d.addCallback(_check)
2130         return d
2131     def failUnlessNodeHasChild(self, node, name):
2132         assert isinstance(name, unicode)
2133         d = node.list()
2134         def _check(children):
2135             self.failUnlessIn(name, children)
2136         d.addCallback(_check)
2137         return d
2138     def failIfNodeHasChild(self, node, name):
2139         assert isinstance(name, unicode)
2140         d = node.list()
2141         def _check(children):
2142             self.failIfIn(name, children)
2143         d.addCallback(_check)
2144         return d
2145
2146     def failUnlessChildContentsAre(self, node, name, expected_contents):
2147         assert isinstance(name, unicode)
2148         d = node.get_child_at_path(name)
2149         d.addCallback(lambda node: download_to_data(node))
2150         def _check(contents):
2151             self.failUnlessReallyEqual(contents, expected_contents)
2152         d.addCallback(_check)
2153         return d
2154
2155     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2156         assert isinstance(name, unicode)
2157         d = node.get_child_at_path(name)
2158         d.addCallback(lambda node: node.download_best_version())
2159         def _check(contents):
2160             self.failUnlessReallyEqual(contents, expected_contents)
2161         d.addCallback(_check)
2162         return d
2163
2164     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2165         assert isinstance(name, unicode)
2166         d = node.get_child_at_path(name)
2167         def _check(child):
2168             self.failUnless(child.is_unknown() or not child.is_readonly())
2169             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2170             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2171             expected_ro_uri = self._make_readonly(expected_uri)
2172             if expected_ro_uri:
2173                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2174         d.addCallback(_check)
2175         return d
2176
2177     def failUnlessROChildURIIs(self, node, name, expected_uri):
2178         assert isinstance(name, unicode)
2179         d = node.get_child_at_path(name)
2180         def _check(child):
2181             self.failUnless(child.is_unknown() or child.is_readonly())
2182             self.failUnlessReallyEqual(child.get_write_uri(), None)
2183             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2184             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2185         d.addCallback(_check)
2186         return d
2187
2188     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2189         assert isinstance(name, unicode)
2190         d = node.get_child_at_path(name)
2191         def _check(child):
2192             self.failUnless(child.is_unknown() or not child.is_readonly())
2193             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2194             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2195             expected_ro_uri = self._make_readonly(got_uri)
2196             if expected_ro_uri:
2197                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2198         d.addCallback(_check)
2199         return d
2200
2201     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2202         assert isinstance(name, unicode)
2203         d = node.get_child_at_path(name)
2204         def _check(child):
2205             self.failUnless(child.is_unknown() or child.is_readonly())
2206             self.failUnlessReallyEqual(child.get_write_uri(), None)
2207             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2208             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2209         d.addCallback(_check)
2210         return d
2211
2212     def failUnlessCHKURIHasContents(self, got_uri, contents):
2213         self.failUnless(self.get_all_contents()[got_uri] == contents)
2214
2215     def test_POST_upload(self):
2216         d = self.POST(self.public_url + "/foo", t="upload",
2217                       file=("new.txt", self.NEWFILE_CONTENTS))
2218         fn = self._foo_node
2219         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2220         d.addCallback(lambda res:
2221                       self.failUnlessChildContentsAre(fn, u"new.txt",
2222                                                       self.NEWFILE_CONTENTS))
2223         return d
2224
2225     def test_POST_upload_unicode(self):
2226         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2227         d = self.POST(self.public_url + "/foo", t="upload",
2228                       file=(filename, self.NEWFILE_CONTENTS))
2229         fn = self._foo_node
2230         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2231         d.addCallback(lambda res:
2232                       self.failUnlessChildContentsAre(fn, filename,
2233                                                       self.NEWFILE_CONTENTS))
2234         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2235         d.addCallback(lambda res: self.GET(target_url))
2236         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2237                                                                   self.NEWFILE_CONTENTS,
2238                                                                   contents))
2239         return d
2240
2241     def test_POST_upload_unicode_named(self):
2242         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2243         d = self.POST(self.public_url + "/foo", t="upload",
2244                       name=filename,
2245                       file=("overridden", self.NEWFILE_CONTENTS))
2246         fn = self._foo_node
2247         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2248         d.addCallback(lambda res:
2249                       self.failUnlessChildContentsAre(fn, filename,
2250                                                       self.NEWFILE_CONTENTS))
2251         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2252         d.addCallback(lambda res: self.GET(target_url))
2253         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2254                                                                   self.NEWFILE_CONTENTS,
2255                                                                   contents))
2256         return d
2257
2258     def test_POST_upload_no_link(self):
2259         d = self.POST("/uri", t="upload",
2260                       file=("new.txt", self.NEWFILE_CONTENTS))
2261         def _check_upload_results(page):
2262             # this should be a page which describes the results of the upload
2263             # that just finished.
2264             self.failUnlessIn("Upload Results:", page)
2265             self.failUnlessIn("URI:", page)
2266             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2267             mo = uri_re.search(page)
2268             self.failUnless(mo, page)
2269             new_uri = mo.group(1)
2270             return new_uri
2271         d.addCallback(_check_upload_results)
2272         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2273         return d
2274
2275     def test_POST_upload_no_link_whendone(self):
2276         d = self.POST("/uri", t="upload", when_done="/",
2277                       file=("new.txt", self.NEWFILE_CONTENTS))
2278         d.addBoth(self.shouldRedirect, "/")
2279         return d
2280
2281     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2282         d = defer.maybeDeferred(callable, *args, **kwargs)
2283         def done(res):
2284             if isinstance(res, failure.Failure):
2285                 res.trap(error.PageRedirect)
2286                 statuscode = res.value.status
2287                 target = res.value.location
2288                 return checker(statuscode, target)
2289             self.fail("%s: callable was supposed to redirect, not return '%s'"
2290                       % (which, res))
2291         d.addBoth(done)
2292         return d
2293
2294     def test_POST_upload_no_link_whendone_results(self):
2295         def check(statuscode, target):
2296             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2297             self.failUnless(target.startswith(self.webish_url), target)
2298             return client.getPage(target, method="GET")
2299         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2300         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2301                                  check,
2302                                  self.POST, "/uri", t="upload",
2303                                  when_done="/%75ri/%(uri)s",
2304                                  file=("new.txt", self.NEWFILE_CONTENTS))
2305         d.addCallback(lambda res:
2306                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2307         return d
2308
2309     def test_POST_upload_no_link_mutable(self):
2310         d = self.POST("/uri", t="upload", mutable="true",
2311                       file=("new.txt", self.NEWFILE_CONTENTS))
2312         def _check(filecap):
2313             filecap = filecap.strip()
2314             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2315             self.filecap = filecap
2316             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2317             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2318             n = self.s.create_node_from_uri(filecap)
2319             return n.download_best_version()
2320         d.addCallback(_check)
2321         def _check2(data):
2322             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2323             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2324         d.addCallback(_check2)
2325         def _check3(data):
2326             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2327             return self.GET("/file/%s" % urllib.quote(self.filecap))
2328         d.addCallback(_check3)
2329         def _check4(data):
2330             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2331         d.addCallback(_check4)
2332         return d
2333
2334     def test_POST_upload_no_link_mutable_toobig(self):
2335         # The SDMF size limit is no longer in place, so we should be
2336         # able to upload mutable files that are as large as we want them
2337         # to be.
2338         d = self.POST("/uri", t="upload", mutable="true",
2339                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2340         return d
2341
2342
2343     def test_POST_upload_format_unlinked(self):
2344         def _check_upload_unlinked(ign, format, uri_prefix):
2345             filename = format + ".txt"
2346             d = self.POST("/uri?t=upload&format=" + format,
2347                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2348             def _got_results(results):
2349                 if format.upper() in ("SDMF", "MDMF"):
2350                     # webapi.rst says this returns a filecap
2351                     filecap = results
2352                 else:
2353                     # for immutable, it returns an "upload results page", and
2354                     # the filecap is buried inside
2355                     line = [l for l in results.split("\n") if "URI: " in l][0]
2356                     mo = re.search(r'<span>([^<]+)</span>', line)
2357                     filecap = mo.group(1)
2358                 self.failUnless(filecap.startswith(uri_prefix),
2359                                 (uri_prefix, filecap))
2360                 return self.GET("/uri/%s?t=json" % filecap)
2361             d.addCallback(_got_results)
2362             def _got_json(json):
2363                 data = simplejson.loads(json)
2364                 data = data[1]
2365                 self.failUnlessIn("format", data)
2366                 self.failUnlessEqual(data["format"], format.upper())
2367             d.addCallback(_got_json)
2368             return d
2369         d = defer.succeed(None)
2370         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2371         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2372         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2373         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2374         return d
2375
2376     def test_POST_upload_bad_format_unlinked(self):
2377         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2378                                     400, "Bad Request", "Unknown format: foo",
2379                                     self.POST,
2380                                     "/uri?t=upload&format=foo",
2381                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2382
2383     def test_POST_upload_format(self):
2384         def _check_upload(ign, format, uri_prefix, fn=None):
2385             filename = format + ".txt"
2386             d = self.POST(self.public_url +
2387                           "/foo?t=upload&format=" + format,
2388                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2389             def _got_filecap(filecap):
2390                 if fn is not None:
2391                     filenameu = unicode(filename)
2392                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2393                 self.failUnless(filecap.startswith(uri_prefix))
2394                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2395             d.addCallback(_got_filecap)
2396             def _got_json(json):
2397                 data = simplejson.loads(json)
2398                 data = data[1]
2399                 self.failUnlessIn("format", data)
2400                 self.failUnlessEqual(data["format"], format.upper())
2401             d.addCallback(_got_json)
2402             return d
2403
2404         d = defer.succeed(None)
2405         d.addCallback(_check_upload, "chk", "URI:CHK")
2406         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2407         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2408         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2409         return d
2410
2411     def test_POST_upload_bad_format(self):
2412         return self.shouldHTTPError("POST_upload_bad_format",
2413                                     400, "Bad Request", "Unknown format: foo",
2414                                     self.POST, self.public_url + \
2415                                     "/foo?t=upload&format=foo",
2416                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2417
2418     def test_POST_upload_mutable(self):
2419         # this creates a mutable file
2420         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2421                       file=("new.txt", self.NEWFILE_CONTENTS))
2422         fn = self._foo_node
2423         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2424         d.addCallback(lambda res:
2425                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2426                                                              self.NEWFILE_CONTENTS))
2427         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2428         def _got(newnode):
2429             self.failUnless(IMutableFileNode.providedBy(newnode))
2430             self.failUnless(newnode.is_mutable())
2431             self.failIf(newnode.is_readonly())
2432             self._mutable_node = newnode
2433             self._mutable_uri = newnode.get_uri()
2434         d.addCallback(_got)
2435
2436         # now upload it again and make sure that the URI doesn't change
2437         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2438         d.addCallback(lambda res:
2439                       self.POST(self.public_url + "/foo", t="upload",
2440                                 mutable="true",
2441                                 file=("new.txt", NEWER_CONTENTS)))
2442         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2443         d.addCallback(lambda res:
2444                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2445                                                              NEWER_CONTENTS))
2446         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2447         def _got2(newnode):
2448             self.failUnless(IMutableFileNode.providedBy(newnode))
2449             self.failUnless(newnode.is_mutable())
2450             self.failIf(newnode.is_readonly())
2451             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2452         d.addCallback(_got2)
2453
2454         # upload a second time, using PUT instead of POST
2455         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2456         d.addCallback(lambda res:
2457                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2458         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2459         d.addCallback(lambda res:
2460                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2461                                                              NEW2_CONTENTS))
2462
2463         # finally list the directory, since mutable files are displayed
2464         # slightly differently
2465
2466         d.addCallback(lambda res:
2467                       self.GET(self.public_url + "/foo/",
2468                                followRedirect=True))
2469         def _check_page(res):
2470             # TODO: assert more about the contents
2471             self.failUnlessIn("SSK", res)
2472             return res
2473         d.addCallback(_check_page)
2474
2475         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2476         def _got3(newnode):
2477             self.failUnless(IMutableFileNode.providedBy(newnode))
2478             self.failUnless(newnode.is_mutable())
2479             self.failIf(newnode.is_readonly())
2480             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2481         d.addCallback(_got3)
2482
2483         # look at the JSON form of the enclosing directory
2484         d.addCallback(lambda res:
2485                       self.GET(self.public_url + "/foo/?t=json",
2486                                followRedirect=True))
2487         def _check_page_json(res):
2488             parsed = simplejson.loads(res)
2489             self.failUnlessEqual(parsed[0], "dirnode")
2490             children = dict( [(unicode(name),value)
2491                               for (name,value)
2492                               in parsed[1]["children"].iteritems()] )
2493             self.failUnlessIn(u"new.txt", children)
2494             new_json = children[u"new.txt"]
2495             self.failUnlessEqual(new_json[0], "filenode")
2496             self.failUnless(new_json[1]["mutable"])
2497             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2498             ro_uri = self._mutable_node.get_readonly().to_string()
2499             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2500         d.addCallback(_check_page_json)
2501
2502         # and the JSON form of the file
2503         d.addCallback(lambda res:
2504                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2505         def _check_file_json(res):
2506             parsed = simplejson.loads(res)
2507             self.failUnlessEqual(parsed[0], "filenode")
2508             self.failUnless(parsed[1]["mutable"])
2509             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2510             ro_uri = self._mutable_node.get_readonly().to_string()
2511             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2512         d.addCallback(_check_file_json)
2513
2514         # and look at t=uri and t=readonly-uri
2515         d.addCallback(lambda res:
2516                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2517         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2518         d.addCallback(lambda res:
2519                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2520         def _check_ro_uri(res):
2521             ro_uri = self._mutable_node.get_readonly().to_string()
2522             self.failUnlessReallyEqual(res, ro_uri)
2523         d.addCallback(_check_ro_uri)
2524
2525         # make sure we can get to it from /uri/URI
2526         d.addCallback(lambda res:
2527                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2528         d.addCallback(lambda res:
2529                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2530
2531         # and that HEAD computes the size correctly
2532         d.addCallback(lambda res:
2533                       self.HEAD(self.public_url + "/foo/new.txt",
2534                                 return_response=True))
2535         def _got_headers((res, status, headers)):
2536             self.failUnlessReallyEqual(res, "")
2537             self.failUnlessReallyEqual(headers["content-length"][0],
2538                                        str(len(NEW2_CONTENTS)))
2539             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2540         d.addCallback(_got_headers)
2541
2542         # make sure that outdated size limits aren't enforced anymore.
2543         d.addCallback(lambda ignored:
2544             self.POST(self.public_url + "/foo", t="upload",
2545                       mutable="true",
2546                       file=("new.txt",
2547                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2548         d.addErrback(self.dump_error)
2549         return d
2550
2551     def test_POST_upload_mutable_toobig(self):
2552         # SDMF had a size limti that was removed a while ago. MDMF has
2553         # never had a size limit. Test to make sure that we do not
2554         # encounter errors when trying to upload large mutable files,
2555         # since there should be no coded prohibitions regarding large
2556         # mutable files.
2557         d = self.POST(self.public_url + "/foo",
2558                       t="upload", mutable="true",
2559                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2560         return d
2561
2562     def dump_error(self, f):
2563         # if the web server returns an error code (like 400 Bad Request),
2564         # web.client.getPage puts the HTTP response body into the .response
2565         # attribute of the exception object that it gives back. It does not
2566         # appear in the Failure's repr(), so the ERROR that trial displays
2567         # will be rather terse and unhelpful. addErrback this method to the
2568         # end of your chain to get more information out of these errors.
2569         if f.check(error.Error):
2570             print "web.error.Error:"
2571             print f
2572             print f.value.response
2573         return f
2574
2575     def test_POST_upload_replace(self):
2576         d = self.POST(self.public_url + "/foo", t="upload",
2577                       file=("bar.txt", self.NEWFILE_CONTENTS))
2578         fn = self._foo_node
2579         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2580         d.addCallback(lambda res:
2581                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2582                                                       self.NEWFILE_CONTENTS))
2583         return d
2584
2585     def test_POST_upload_no_replace_ok(self):
2586         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2587                       file=("new.txt", self.NEWFILE_CONTENTS))
2588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2589         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2590                                                              self.NEWFILE_CONTENTS))
2591         return d
2592
2593     def test_POST_upload_no_replace_queryarg(self):
2594         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2595                       file=("bar.txt", self.NEWFILE_CONTENTS))
2596         d.addBoth(self.shouldFail, error.Error,
2597                   "POST_upload_no_replace_queryarg",
2598                   "409 Conflict",
2599                   "There was already a child by that name, and you asked me "
2600                   "to not replace it")
2601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2602         d.addCallback(self.failUnlessIsBarDotTxt)
2603         return d
2604
2605     def test_POST_upload_no_replace_field(self):
2606         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2607                       file=("bar.txt", self.NEWFILE_CONTENTS))
2608         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2609                   "409 Conflict",
2610                   "There was already a child by that name, and you asked me "
2611                   "to not replace it")
2612         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2613         d.addCallback(self.failUnlessIsBarDotTxt)
2614         return d
2615
2616     def test_POST_upload_whendone(self):
2617         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2618                       file=("new.txt", self.NEWFILE_CONTENTS))
2619         d.addBoth(self.shouldRedirect, "/THERE")
2620         fn = self._foo_node
2621         d.addCallback(lambda res:
2622                       self.failUnlessChildContentsAre(fn, u"new.txt",
2623                                                       self.NEWFILE_CONTENTS))
2624         return d
2625
2626     def test_POST_upload_named(self):
2627         fn = self._foo_node
2628         d = self.POST(self.public_url + "/foo", t="upload",
2629                       name="new.txt", file=self.NEWFILE_CONTENTS)
2630         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2631         d.addCallback(lambda res:
2632                       self.failUnlessChildContentsAre(fn, u"new.txt",
2633                                                       self.NEWFILE_CONTENTS))
2634         return d
2635
2636     def test_POST_upload_named_badfilename(self):
2637         d = self.POST(self.public_url + "/foo", t="upload",
2638                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2639         d.addBoth(self.shouldFail, error.Error,
2640                   "test_POST_upload_named_badfilename",
2641                   "400 Bad Request",
2642                   "name= may not contain a slash",
2643                   )
2644         # make sure that nothing was added
2645         d.addCallback(lambda res:
2646                       self.failUnlessNodeKeysAre(self._foo_node,
2647                                                  [self._htmlname_unicode,
2648                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2649                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2650                                                   u"sub"]))
2651         return d
2652
2653     def test_POST_FILEURL_check(self):
2654         bar_url = self.public_url + "/foo/bar.txt"
2655         d = self.POST(bar_url, t="check")
2656         def _check(res):
2657             self.failUnlessIn("Healthy :", res)
2658         d.addCallback(_check)
2659         redir_url = "http://allmydata.org/TARGET"
2660         def _check2(statuscode, target):
2661             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2662             self.failUnlessReallyEqual(target, redir_url)
2663         d.addCallback(lambda res:
2664                       self.shouldRedirect2("test_POST_FILEURL_check",
2665                                            _check2,
2666                                            self.POST, bar_url,
2667                                            t="check",
2668                                            when_done=redir_url))
2669         d.addCallback(lambda res:
2670                       self.POST(bar_url, t="check", return_to=redir_url))
2671         def _check3(res):
2672             self.failUnlessIn("Healthy :", res)
2673             self.failUnlessIn("Return to file", res)
2674             self.failUnlessIn(redir_url, res)
2675         d.addCallback(_check3)
2676
2677         d.addCallback(lambda res:
2678                       self.POST(bar_url, t="check", output="JSON"))
2679         def _check_json(res):
2680             data = simplejson.loads(res)
2681             self.failUnlessIn("storage-index", data)
2682             self.failUnless(data["results"]["healthy"])
2683         d.addCallback(_check_json)
2684
2685         return d
2686
2687     def test_POST_FILEURL_check_and_repair(self):
2688         bar_url = self.public_url + "/foo/bar.txt"
2689         d = self.POST(bar_url, t="check", repair="true")
2690         def _check(res):
2691             self.failUnlessIn("Healthy :", res)
2692         d.addCallback(_check)
2693         redir_url = "http://allmydata.org/TARGET"
2694         def _check2(statuscode, target):
2695             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2696             self.failUnlessReallyEqual(target, redir_url)
2697         d.addCallback(lambda res:
2698                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2699                                            _check2,
2700                                            self.POST, bar_url,
2701                                            t="check", repair="true",
2702                                            when_done=redir_url))
2703         d.addCallback(lambda res:
2704                       self.POST(bar_url, t="check", return_to=redir_url))
2705         def _check3(res):
2706             self.failUnlessIn("Healthy :", res)
2707             self.failUnlessIn("Return to file", res)
2708             self.failUnlessIn(redir_url, res)
2709         d.addCallback(_check3)
2710         return d
2711
2712     def test_POST_DIRURL_check(self):
2713         foo_url = self.public_url + "/foo/"
2714         d = self.POST(foo_url, t="check")
2715         def _check(res):
2716             self.failUnlessIn("Healthy :", res)
2717         d.addCallback(_check)
2718         redir_url = "http://allmydata.org/TARGET"
2719         def _check2(statuscode, target):
2720             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2721             self.failUnlessReallyEqual(target, redir_url)
2722         d.addCallback(lambda res:
2723                       self.shouldRedirect2("test_POST_DIRURL_check",
2724                                            _check2,
2725                                            self.POST, foo_url,
2726                                            t="check",
2727                                            when_done=redir_url))
2728         d.addCallback(lambda res:
2729                       self.POST(foo_url, t="check", return_to=redir_url))
2730         def _check3(res):
2731             self.failUnlessIn("Healthy :", res)
2732             self.failUnlessIn("Return to file/directory", res)
2733             self.failUnlessIn(redir_url, res)
2734         d.addCallback(_check3)
2735
2736         d.addCallback(lambda res:
2737                       self.POST(foo_url, t="check", output="JSON"))
2738         def _check_json(res):
2739             data = simplejson.loads(res)
2740             self.failUnlessIn("storage-index", data)
2741             self.failUnless(data["results"]["healthy"])
2742         d.addCallback(_check_json)
2743
2744         return d
2745
2746     def test_POST_DIRURL_check_and_repair(self):
2747         foo_url = self.public_url + "/foo/"
2748         d = self.POST(foo_url, t="check", repair="true")
2749         def _check(res):
2750             self.failUnlessIn("Healthy :", res)
2751         d.addCallback(_check)
2752         redir_url = "http://allmydata.org/TARGET"
2753         def _check2(statuscode, target):
2754             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2755             self.failUnlessReallyEqual(target, redir_url)
2756         d.addCallback(lambda res:
2757                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2758                                            _check2,
2759                                            self.POST, foo_url,
2760                                            t="check", repair="true",
2761                                            when_done=redir_url))
2762         d.addCallback(lambda res:
2763                       self.POST(foo_url, t="check", return_to=redir_url))
2764         def _check3(res):
2765             self.failUnlessIn("Healthy :", res)
2766             self.failUnlessIn("Return to file/directory", res)
2767             self.failUnlessIn(redir_url, res)
2768         d.addCallback(_check3)
2769         return d
2770
2771     def test_POST_FILEURL_mdmf_check(self):
2772         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2773         d = self.POST(quux_url, t="check")
2774         def _check(res):
2775             self.failUnlessIn("Healthy", res)
2776         d.addCallback(_check)
2777         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2778         d.addCallback(lambda ignored:
2779                       self.POST(quux_extension_url, t="check"))
2780         d.addCallback(_check)
2781         return d
2782
2783     def test_POST_FILEURL_mdmf_check_and_repair(self):
2784         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2785         d = self.POST(quux_url, t="check", repair="true")
2786         def _check(res):
2787             self.failUnlessIn("Healthy", res)
2788         d.addCallback(_check)
2789         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2790         d.addCallback(lambda ignored:
2791                       self.POST(quux_extension_url, t="check", repair="true"))
2792         d.addCallback(_check)
2793         return d
2794
2795     def wait_for_operation(self, ignored, ophandle):
2796         url = "/operations/" + ophandle
2797         url += "?t=status&output=JSON"
2798         d = self.GET(url)
2799         def _got(res):
2800             data = simplejson.loads(res)
2801             if not data["finished"]:
2802                 d = self.stall(delay=1.0)
2803                 d.addCallback(self.wait_for_operation, ophandle)
2804                 return d
2805             return data
2806         d.addCallback(_got)
2807         return d
2808
2809     def get_operation_results(self, ignored, ophandle, output=None):
2810         url = "/operations/" + ophandle
2811         url += "?t=status"
2812         if output:
2813             url += "&output=" + output
2814         d = self.GET(url)
2815         def _got(res):
2816             if output and output.lower() == "json":
2817                 return simplejson.loads(res)
2818             return res
2819         d.addCallback(_got)
2820         return d
2821
2822     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2823         d = self.shouldFail2(error.Error,
2824                              "test_POST_DIRURL_deepcheck_no_ophandle",
2825                              "400 Bad Request",
2826                              "slow operation requires ophandle=",
2827                              self.POST, self.public_url, t="start-deep-check")
2828         return d
2829
2830     def test_POST_DIRURL_deepcheck(self):
2831         def _check_redirect(statuscode, target):
2832             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2833             self.failUnless(target.endswith("/operations/123"))
2834         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2835                                  self.POST, self.public_url,
2836                                  t="start-deep-check", ophandle="123")
2837         d.addCallback(self.wait_for_operation, "123")
2838         def _check_json(data):
2839             self.failUnlessReallyEqual(data["finished"], True)
2840             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2841             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2842         d.addCallback(_check_json)
2843         d.addCallback(self.get_operation_results, "123", "html")
2844         def _check_html(res):
2845             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2846             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2847             self.failUnlessIn(FAVICON_MARKUP, res)
2848         d.addCallback(_check_html)
2849
2850         d.addCallback(lambda res:
2851                       self.GET("/operations/123/"))
2852         d.addCallback(_check_html) # should be the same as without the slash
2853
2854         d.addCallback(lambda res:
2855                       self.shouldFail2(error.Error, "one", "404 Not Found",
2856                                        "No detailed results for SI bogus",
2857                                        self.GET, "/operations/123/bogus"))
2858
2859         foo_si = self._foo_node.get_storage_index()
2860         foo_si_s = base32.b2a(foo_si)
2861         d.addCallback(lambda res:
2862                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2863         def _check_foo_json(res):
2864             data = simplejson.loads(res)
2865             self.failUnlessEqual(data["storage-index"], foo_si_s)
2866             self.failUnless(data["results"]["healthy"])
2867         d.addCallback(_check_foo_json)
2868         return d
2869
2870     def test_POST_DIRURL_deepcheck_and_repair(self):
2871         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2872                       ophandle="124", output="json", followRedirect=True)
2873         d.addCallback(self.wait_for_operation, "124")
2874         def _check_json(data):
2875             self.failUnlessReallyEqual(data["finished"], True)
2876             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2877             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2878             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2879             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2880             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2881             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2882             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2883             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2884             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2885             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2886         d.addCallback(_check_json)
2887         d.addCallback(self.get_operation_results, "124", "html")
2888         def _check_html(res):
2889             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2890
2891             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2892             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2893             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2894
2895             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2896             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2897             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2898
2899             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2900             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2901             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2902
2903             self.failUnlessIn(FAVICON_MARKUP, res)
2904         d.addCallback(_check_html)
2905         return d
2906
2907     def test_POST_FILEURL_bad_t(self):
2908         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2909                              "POST to file: bad t=bogus",
2910                              self.POST, self.public_url + "/foo/bar.txt",
2911                              t="bogus")
2912         return d
2913
2914     def test_POST_mkdir(self): # return value?
2915         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2916         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917         d.addCallback(self.failUnlessNodeKeysAre, [])
2918         return d
2919
2920     def test_POST_mkdir_mdmf(self):
2921         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2922         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2923         d.addCallback(lambda node:
2924             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2925         return d
2926
2927     def test_POST_mkdir_sdmf(self):
2928         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2929         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2930         d.addCallback(lambda node:
2931             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2932         return d
2933
2934     def test_POST_mkdir_bad_format(self):
2935         return self.shouldHTTPError("POST_mkdir_bad_format",
2936                                     400, "Bad Request", "Unknown format: foo",
2937                                     self.POST, self.public_url +
2938                                     "/foo?t=mkdir&name=newdir&format=foo")
2939
2940     def test_POST_mkdir_initial_children(self):
2941         (newkids, caps) = self._create_initial_children()
2942         d = self.POST2(self.public_url +
2943                        "/foo?t=mkdir-with-children&name=newdir",
2944                        simplejson.dumps(newkids))
2945         d.addCallback(lambda res:
2946                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2947         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2949         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2950         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2951         return d
2952
2953     def test_POST_mkdir_initial_children_mdmf(self):
2954         (newkids, caps) = self._create_initial_children()
2955         d = self.POST2(self.public_url +
2956                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2957                        simplejson.dumps(newkids))
2958         d.addCallback(lambda res:
2959                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2960         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2961         d.addCallback(lambda node:
2962             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2963         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2965                        caps['filecap1'])
2966         return d
2967
2968     # XXX: Duplication.
2969     def test_POST_mkdir_initial_children_sdmf(self):
2970         (newkids, caps) = self._create_initial_children()
2971         d = self.POST2(self.public_url +
2972                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2973                        simplejson.dumps(newkids))
2974         d.addCallback(lambda res:
2975                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2976         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2977         d.addCallback(lambda node:
2978             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2979         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2980         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2981                        caps['filecap1'])
2982         return d
2983
2984     def test_POST_mkdir_initial_children_bad_format(self):
2985         (newkids, caps) = self._create_initial_children()
2986         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2987                                     400, "Bad Request", "Unknown format: foo",
2988                                     self.POST, self.public_url + \
2989                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2990                                     simplejson.dumps(newkids))
2991
2992     def test_POST_mkdir_immutable(self):
2993         (newkids, caps) = self._create_immutable_children()
2994         d = self.POST2(self.public_url +
2995                        "/foo?t=mkdir-immutable&name=newdir",
2996                        simplejson.dumps(newkids))
2997         d.addCallback(lambda res:
2998                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
3001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3007         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3008         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3009         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3010         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3011         return d
3012
3013     def test_POST_mkdir_immutable_bad(self):
3014         (newkids, caps) = self._create_initial_children()
3015         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3016                              "400 Bad Request",
3017                              "needed to be immutable but was not",
3018                              self.POST2,
3019                              self.public_url +
3020                              "/foo?t=mkdir-immutable&name=newdir",
3021                              simplejson.dumps(newkids))
3022         return d
3023
3024     def test_POST_mkdir_2(self):
3025         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3026         d.addCallback(lambda res:
3027                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3028         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3029         d.addCallback(self.failUnlessNodeKeysAre, [])
3030         return d
3031
3032     def test_POST_mkdirs_2(self):
3033         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3034         d.addCallback(lambda res:
3035                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3036         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3037         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3038         d.addCallback(self.failUnlessNodeKeysAre, [])
3039         return d
3040
3041     def test_POST_mkdir_no_parentdir_noredirect(self):
3042         d = self.POST("/uri?t=mkdir")
3043         def _after_mkdir(res):
3044             uri.DirectoryURI.init_from_string(res)
3045         d.addCallback(_after_mkdir)
3046         return d
3047
3048     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3049         d = self.POST("/uri?t=mkdir&format=mdmf")
3050         def _after_mkdir(res):
3051             u = uri.from_string(res)
3052             # Check that this is an MDMF writecap
3053             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3054         d.addCallback(_after_mkdir)
3055         return d
3056
3057     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3058         d = self.POST("/uri?t=mkdir&format=sdmf")
3059         def _after_mkdir(res):
3060             u = uri.from_string(res)
3061             self.failUnlessIsInstance(u, uri.DirectoryURI)
3062         d.addCallback(_after_mkdir)
3063         return d
3064
3065     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3066         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3067                                     400, "Bad Request", "Unknown format: foo",
3068                                     self.POST, self.public_url +
3069                                     "/uri?t=mkdir&format=foo")
3070
3071     def test_POST_mkdir_no_parentdir_noredirect2(self):
3072         # make sure form-based arguments (as on the welcome page) still work
3073         d = self.POST("/uri", t="mkdir")
3074         def _after_mkdir(res):
3075             uri.DirectoryURI.init_from_string(res)
3076         d.addCallback(_after_mkdir)
3077         d.addErrback(self.explain_web_error)
3078         return d
3079
3080     def test_POST_mkdir_no_parentdir_redirect(self):
3081         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3082         d.addBoth(self.shouldRedirect, None, statuscode='303')
3083         def _check_target(target):
3084             target = urllib.unquote(target)
3085             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3086         d.addCallback(_check_target)
3087         return d
3088
3089     def test_POST_mkdir_no_parentdir_redirect2(self):
3090         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3091         d.addBoth(self.shouldRedirect, None, statuscode='303')
3092         def _check_target(target):
3093             target = urllib.unquote(target)
3094             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3095         d.addCallback(_check_target)
3096         d.addErrback(self.explain_web_error)
3097         return d
3098
3099     def _make_readonly(self, u):
3100         ro_uri = uri.from_string(u).get_readonly()
3101         if ro_uri is None:
3102             return None
3103         return ro_uri.to_string()
3104
3105     def _create_initial_children(self):
3106         contents, n, filecap1 = self.makefile(12)
3107         md1 = {"metakey1": "metavalue1"}
3108         filecap2 = make_mutable_file_uri()
3109         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3110         filecap3 = node3.get_readonly_uri()
3111         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3112         dircap = DirectoryNode(node4, None, None).get_uri()
3113         mdmfcap = make_mutable_file_uri(mdmf=True)
3114         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3115         emptydircap = "URI:DIR2-LIT:"
3116         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3117                                                       "ro_uri": self._make_readonly(filecap1),
3118                                                       "metadata": md1, }],
3119                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3120                                                       "ro_uri": self._make_readonly(filecap2)}],
3121                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3122                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3123                                                       "ro_uri": unknown_rocap}],
3124                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3125                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3126                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3127                                                       "ro_uri": self._make_readonly(dircap)}],
3128                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3129                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3130                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3131                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3132                    }
3133         return newkids, {'filecap1': filecap1,
3134                          'filecap2': filecap2,
3135                          'filecap3': filecap3,
3136                          'unknown_rwcap': unknown_rwcap,
3137                          'unknown_rocap': unknown_rocap,
3138                          'unknown_immcap': unknown_immcap,
3139                          'dircap': dircap,
3140                          'litdircap': litdircap,
3141                          'emptydircap': emptydircap,
3142                          'mdmfcap': mdmfcap}
3143
3144     def _create_immutable_children(self):
3145         contents, n, filecap1 = self.makefile(12)
3146         md1 = {"metakey1": "metavalue1"}
3147         tnode = create_chk_filenode("immutable directory contents\n"*10,
3148                                     self.get_all_contents())
3149         dnode = DirectoryNode(tnode, None, None)
3150         assert not dnode.is_mutable()
3151         immdircap = dnode.get_uri()
3152         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3153         emptydircap = "URI:DIR2-LIT:"
3154         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3155                                                       "metadata": md1, }],
3156                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3157                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3158                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3159                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3160                    }
3161         return newkids, {'filecap1': filecap1,
3162                          'unknown_immcap': unknown_immcap,
3163                          'immdircap': immdircap,
3164                          'litdircap': litdircap,
3165                          'emptydircap': emptydircap}
3166
3167     def test_POST_mkdir_no_parentdir_initial_children(self):
3168         (newkids, caps) = self._create_initial_children()
3169         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3170         def _after_mkdir(res):
3171             self.failUnless(res.startswith("URI:DIR"), res)
3172             n = self.s.create_node_from_uri(res)
3173             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3174             d2.addCallback(lambda ign:
3175                            self.failUnlessROChildURIIs(n, u"child-imm",
3176                                                        caps['filecap1']))
3177             d2.addCallback(lambda ign:
3178                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3179                                                        caps['filecap2']))
3180             d2.addCallback(lambda ign:
3181                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3182                                                        caps['filecap3']))
3183             d2.addCallback(lambda ign:
3184                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3185                                                        caps['unknown_rwcap']))
3186             d2.addCallback(lambda ign:
3187                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3188                                                        caps['unknown_rocap']))
3189             d2.addCallback(lambda ign:
3190                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3191                                                        caps['unknown_immcap']))
3192             d2.addCallback(lambda ign:
3193                            self.failUnlessRWChildURIIs(n, u"dirchild",
3194                                                        caps['dircap']))
3195             return d2
3196         d.addCallback(_after_mkdir)
3197         return d
3198
3199     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3200         # the regular /uri?t=mkdir operation is specified to ignore its body.
3201         # Only t=mkdir-with-children pays attention to it.
3202         (newkids, caps) = self._create_initial_children()
3203         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3204                                  400, "Bad Request",
3205                                  "t=mkdir does not accept children=, "
3206                                  "try t=mkdir-with-children instead",
3207                                  self.POST2, "/uri?t=mkdir", # without children
3208                                  simplejson.dumps(newkids))
3209         return d
3210
3211     def test_POST_noparent_bad(self):
3212         d = self.shouldHTTPError("POST_noparent_bad",
3213                                  400, "Bad Request",
3214                                  "/uri accepts only PUT, PUT?t=mkdir, "
3215                                  "POST?t=upload, and POST?t=mkdir",
3216                                  self.POST, "/uri?t=bogus")
3217         return d
3218
3219     def test_POST_mkdir_no_parentdir_immutable(self):
3220         (newkids, caps) = self._create_immutable_children()
3221         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3222         def _after_mkdir(res):
3223             self.failUnless(res.startswith("URI:DIR"), res)
3224             n = self.s.create_node_from_uri(res)
3225             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3226             d2.addCallback(lambda ign:
3227                            self.failUnlessROChildURIIs(n, u"child-imm",
3228                                                           caps['filecap1']))
3229             d2.addCallback(lambda ign:
3230                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3231                                                           caps['unknown_immcap']))
3232             d2.addCallback(lambda ign:
3233                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3234                                                           caps['immdircap']))
3235             d2.addCallback(lambda ign:
3236                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3237                                                           caps['litdircap']))
3238             d2.addCallback(lambda ign:
3239                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3240                                                           caps['emptydircap']))
3241             return d2
3242         d.addCallback(_after_mkdir)
3243         return d
3244
3245     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3246         (newkids, caps) = self._create_initial_children()
3247         d = self.shouldFail2(error.Error,
3248                              "test_POST_mkdir_no_parentdir_immutable_bad",
3249                              "400 Bad Request",
3250                              "needed to be immutable but was not",
3251                              self.POST2,
3252                              "/uri?t=mkdir-immutable",
3253                              simplejson.dumps(newkids))
3254         return d
3255
3256     def test_welcome_page_mkdir_button(self):
3257         # Fetch the welcome page.
3258         d = self.GET("/")
3259         def _after_get_welcome_page(res):
3260             MKDIR_BUTTON_RE = re.compile(
3261                 '<form action="([^"]*)" method="post".*'
3262                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3263                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3264                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3265             html = res.replace('\n', ' ')
3266             mo = MKDIR_BUTTON_RE.search(html)
3267             self.failUnless(mo, html)
3268             formaction = mo.group(1)
3269             formt = mo.group(2)
3270             formaname = mo.group(3)
3271             formavalue = mo.group(4)
3272             return (formaction, formt, formaname, formavalue)
3273         d.addCallback(_after_get_welcome_page)
3274         def _after_parse_form(res):
3275             (formaction, formt, formaname, formavalue) = res
3276             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3277         d.addCallback(_after_parse_form)
3278         d.addBoth(self.shouldRedirect, None, statuscode='303')
3279         return d
3280
3281     def test_POST_mkdir_replace(self): # return value?
3282         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3283         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3284         d.addCallback(self.failUnlessNodeKeysAre, [])
3285         return d
3286
3287     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3288         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3289         d.addBoth(self.shouldFail, error.Error,
3290                   "POST_mkdir_no_replace_queryarg",
3291                   "409 Conflict",
3292                   "There was already a child by that name, and you asked me "
3293                   "to not replace it")
3294         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3295         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3296         return d
3297
3298     def test_POST_mkdir_no_replace_field(self): # return value?
3299         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3300                       replace="false")
3301         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3302                   "409 Conflict",
3303                   "There was already a child by that name, and you asked me "
3304                   "to not replace it")
3305         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3306         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3307         return d
3308
3309     def test_POST_mkdir_whendone_field(self):
3310         d = self.POST(self.public_url + "/foo",
3311                       t="mkdir", name="newdir", when_done="/THERE")
3312         d.addBoth(self.shouldRedirect, "/THERE")
3313         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3314         d.addCallback(self.failUnlessNodeKeysAre, [])
3315         return d
3316
3317     def test_POST_mkdir_whendone_queryarg(self):
3318         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3319                       t="mkdir", name="newdir")
3320         d.addBoth(self.shouldRedirect, "/THERE")
3321         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3322         d.addCallback(self.failUnlessNodeKeysAre, [])
3323         return d
3324
3325     def test_POST_bad_t(self):
3326         d = self.shouldFail2(error.Error, "POST_bad_t",
3327                              "400 Bad Request",
3328                              "POST to a directory with bad t=BOGUS",
3329                              self.POST, self.public_url + "/foo", t="BOGUS")
3330         return d
3331
3332     def test_POST_set_children(self, command_name="set_children"):
3333         contents9, n9, newuri9 = self.makefile(9)
3334         contents10, n10, newuri10 = self.makefile(10)
3335         contents11, n11, newuri11 = self.makefile(11)
3336
3337         reqbody = """{
3338                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3339                                                 "size": 0,
3340                                                 "metadata": {
3341                                                   "ctime": 1002777696.7564139,
3342                                                   "mtime": 1002777696.7564139
3343                                                  }
3344                                                } ],
3345                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3346                                                 "size": 1,
3347                                                 "metadata": {
3348                                                   "ctime": 1002777696.7564139,
3349                                                   "mtime": 1002777696.7564139
3350                                                  }
3351                                                } ],
3352                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3353                                                 "size": 2,
3354                                                 "metadata": {
3355                                                   "ctime": 1002777696.7564139,
3356                                                   "mtime": 1002777696.7564139
3357                                                  }
3358                                                } ]
3359                     }""" % (newuri9, newuri10, newuri11)
3360
3361         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3362
3363         d = client.getPage(url, method="POST", postdata=reqbody)
3364         def _then(res):
3365             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3366             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3367             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3368
3369         d.addCallback(_then)
3370         d.addErrback(self.dump_error)
3371         return d
3372
3373     def test_POST_set_children_with_hyphen(self):
3374         return self.test_POST_set_children(command_name="set-children")
3375
3376     def test_POST_link_uri(self):
3377         contents, n, newuri = self.makefile(8)
3378         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3379         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3380         d.addCallback(lambda res:
3381                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3382                                                       contents))
3383         return d
3384
3385     def test_POST_link_uri_replace(self):
3386         contents, n, newuri = self.makefile(8)
3387         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3388         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3389         d.addCallback(lambda res:
3390                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3391                                                       contents))
3392         return d
3393
3394     def test_POST_link_uri_unknown_bad(self):
3395         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3396         d.addBoth(self.shouldFail, error.Error,
3397                   "POST_link_uri_unknown_bad",
3398                   "400 Bad Request",
3399                   "unknown cap in a write slot")
3400         return d
3401
3402     def test_POST_link_uri_unknown_ro_good(self):
3403         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3404         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3405         return d
3406
3407     def test_POST_link_uri_unknown_imm_good(self):
3408         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3409         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3410         return d
3411
3412     def test_POST_link_uri_no_replace_queryarg(self):
3413         contents, n, newuri = self.makefile(8)
3414         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3415                       name="bar.txt", uri=newuri)
3416         d.addBoth(self.shouldFail, error.Error,
3417                   "POST_link_uri_no_replace_queryarg",
3418                   "409 Conflict",
3419                   "There was already a child by that name, and you asked me "
3420                   "to not replace it")
3421         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3422         d.addCallback(self.failUnlessIsBarDotTxt)
3423         return d
3424
3425     def test_POST_link_uri_no_replace_field(self):
3426         contents, n, newuri = self.makefile(8)
3427         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3428                       name="bar.txt", uri=newuri)
3429         d.addBoth(self.shouldFail, error.Error,
3430                   "POST_link_uri_no_replace_field",
3431                   "409 Conflict",
3432                   "There was already a child by that name, and you asked me "
3433                   "to not replace it")
3434         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3435         d.addCallback(self.failUnlessIsBarDotTxt)
3436         return d
3437
3438     def test_POST_delete(self, command_name='delete'):
3439         d = self._foo_node.list()
3440         def _check_before(children):
3441             self.failUnlessIn(u"bar.txt", children)
3442         d.addCallback(_check_before)
3443         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3444         d.addCallback(lambda res: self._foo_node.list())
3445         def _check_after(children):
3446             self.failIfIn(u"bar.txt", children)
3447         d.addCallback(_check_after)
3448         return d
3449
3450     def test_POST_unlink(self):
3451         return self.test_POST_delete(command_name='unlink')
3452
3453     def test_POST_rename_file(self):
3454         d = self.POST(self.public_url + "/foo", t="rename",
3455                       from_name="bar.txt", to_name='wibble.txt')
3456         d.addCallback(lambda res:
3457                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3458         d.addCallback(lambda res:
3459                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3460         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3461         d.addCallback(self.failUnlessIsBarDotTxt)
3462         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3463         d.addCallback(self.failUnlessIsBarJSON)
3464         return d
3465
3466     def test_POST_rename_file_redundant(self):
3467         d = self.POST(self.public_url + "/foo", t="rename",
3468                       from_name="bar.txt", to_name='bar.txt')
3469         d.addCallback(lambda res:
3470                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3471         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3472         d.addCallback(self.failUnlessIsBarDotTxt)
3473         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3474         d.addCallback(self.failUnlessIsBarJSON)
3475         return d
3476
3477     def test_POST_rename_file_replace(self):
3478         # rename a file and replace a directory with it
3479         d = self.POST(self.public_url + "/foo", t="rename",
3480                       from_name="bar.txt", to_name='empty')
3481         d.addCallback(lambda res:
3482                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3483         d.addCallback(lambda res:
3484                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3485         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3486         d.addCallback(self.failUnlessIsBarDotTxt)
3487         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3488         d.addCallback(self.failUnlessIsBarJSON)
3489         return d
3490
3491     def test_POST_rename_file_no_replace_queryarg(self):
3492         # rename a file and replace a directory with it
3493         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3494                       from_name="bar.txt", to_name='empty')
3495         d.addBoth(self.shouldFail, error.Error,
3496                   "POST_rename_file_no_replace_queryarg",
3497                   "409 Conflict",
3498                   "There was already a child by that name, and you asked me "
3499                   "to not replace it")
3500         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3501         d.addCallback(self.failUnlessIsEmptyJSON)
3502         return d
3503
3504     def test_POST_rename_file_no_replace_field(self):
3505         # rename a file and replace a directory with it
3506         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3507                       from_name="bar.txt", to_name='empty')
3508         d.addBoth(self.shouldFail, error.Error,
3509                   "POST_rename_file_no_replace_field",
3510                   "409 Conflict",
3511                   "There was already a child by that name, and you asked me "
3512                   "to not replace it")
3513         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3514         d.addCallback(self.failUnlessIsEmptyJSON)
3515         return d
3516
3517     def test_POST_rename_file_no_replace_same_link(self):
3518         d = self.POST(self.public_url + "/foo", t="rename",
3519                       replace="false", from_name="bar.txt", to_name="bar.txt")
3520         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3522         d.addCallback(self.failUnlessIsBarDotTxt)
3523         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3524         d.addCallback(self.failUnlessIsBarJSON)
3525         return d
3526
3527     def test_POST_rename_file_replace_only_files(self):
3528         d = self.POST(self.public_url + "/foo", t="rename",
3529                       replace="only-files", from_name="bar.txt",
3530                       to_name="baz.txt")
3531         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3532         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3533         d.addCallback(self.failUnlessIsBarDotTxt)
3534         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3535         d.addCallback(self.failUnlessIsBarJSON)
3536         return d
3537
3538     def test_POST_rename_file_replace_only_files_conflict(self):
3539         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3540                              "409 Conflict",
3541                              "There was already a child by that name, and you asked me to not replace it.",
3542                              self.POST, self.public_url + "/foo", t="relink",
3543                              replace="only-files", from_name="bar.txt",
3544                              to_name="empty")
3545         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3546         d.addCallback(self.failUnlessIsBarDotTxt)
3547         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3548         d.addCallback(self.failUnlessIsBarJSON)
3549         return d
3550
3551     def failUnlessIsEmptyJSON(self, res):
3552         data = simplejson.loads(res)
3553         self.failUnlessEqual(data[0], "dirnode", data)
3554         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3555
3556     def test_POST_rename_file_to_slash_fail(self):
3557         d = self.POST(self.public_url + "/foo", t="rename",
3558                       from_name="bar.txt", to_name='kirk/spock.txt')
3559         d.addBoth(self.shouldFail, error.Error,
3560                   "test_POST_rename_file_to_slash_fail",
3561                   "400 Bad Request",
3562                   "to_name= may not contain a slash",
3563                   )
3564         d.addCallback(lambda res:
3565                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3566         return d
3567
3568     def test_POST_rename_file_from_slash_fail(self):
3569         d = self.POST(self.public_url + "/foo", t="rename",
3570                       from_name="sub/bar.txt", to_name='spock.txt')
3571         d.addBoth(self.shouldFail, error.Error,
3572                   "test_POST_rename_from_file_slash_fail",
3573                   "400 Bad Request",
3574                   "from_name= may not contain a slash",
3575                   )
3576         d.addCallback(lambda res:
3577                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3578         return d
3579
3580     def test_POST_rename_dir(self):
3581         d = self.POST(self.public_url, t="rename",
3582                       from_name="foo", to_name='plunk')
3583         d.addCallback(lambda res:
3584                       self.failIfNodeHasChild(self.public_root, u"foo"))
3585         d.addCallback(lambda res:
3586                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3587         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3588         d.addCallback(self.failUnlessIsFooJSON)
3589         return d
3590
3591     def test_POST_relink_file(self):
3592         d = self.POST(self.public_url + "/foo", t="relink",
3593                       from_name="bar.txt",
3594                       to_dir=self.public_root.get_uri() + "/foo/sub")
3595         d.addCallback(lambda res:
3596                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3597         d.addCallback(lambda res:
3598                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3599         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3600         d.addCallback(self.failUnlessIsBarDotTxt)
3601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3602         d.addCallback(self.failUnlessIsBarJSON)
3603         return d
3604
3605     def test_POST_relink_file_new_name(self):
3606         d = self.POST(self.public_url + "/foo", t="relink",
3607                       from_name="bar.txt",
3608                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3609         d.addCallback(lambda res:
3610                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3611         d.addCallback(lambda res:
3612                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3613         d.addCallback(lambda res:
3614                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3615         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3616         d.addCallback(self.failUnlessIsBarDotTxt)
3617         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3618         d.addCallback(self.failUnlessIsBarJSON)
3619         return d
3620
3621     def test_POST_relink_file_replace(self):
3622         d = self.POST(self.public_url + "/foo", t="relink",
3623                       from_name="bar.txt",
3624                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3625         d.addCallback(lambda res:
3626                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3627         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3628         d.addCallback(self.failUnlessIsBarDotTxt)
3629         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3630         d.addCallback(self.failUnlessIsBarJSON)
3631         return d
3632
3633     def test_POST_relink_file_no_replace(self):
3634         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3635                              "409 Conflict",
3636                              "There was already a child by that name, and you asked me to not replace it",
3637                              self.POST, self.public_url + "/foo", t="relink",
3638                              replace="false", from_name="bar.txt",
3639                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3640         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3641         d.addCallback(self.failUnlessIsBarDotTxt)
3642         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3643         d.addCallback(self.failUnlessIsBarJSON)
3644         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3645         d.addCallback(self.failUnlessIsSubBazDotTxt)
3646         return d
3647
3648     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3649         d = self.POST(self.public_url + "/foo", t="relink",
3650                       replace="false", from_name="bar.txt",
3651                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3652         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3653         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3654         d.addCallback(self.failUnlessIsBarDotTxt)
3655         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3656         d.addCallback(self.failUnlessIsBarJSON)
3657         return d
3658
3659     def test_POST_relink_file_replace_only_files(self):
3660         d = self.POST(self.public_url + "/foo", t="relink",
3661                       replace="only-files", from_name="bar.txt",
3662                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3663         d.addCallback(lambda res:
3664                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3665         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3666         d.addCallback(self.failUnlessIsBarDotTxt)
3667         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3668         d.addCallback(self.failUnlessIsBarJSON)
3669         return d
3670
3671     def test_POST_relink_file_replace_only_files_conflict(self):
3672         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3673                              "409 Conflict",
3674                              "There was already a child by that name, and you asked me to not replace it.",
3675                              self.POST, self.public_url + "/foo", t="relink",
3676                              replace="only-files", from_name="bar.txt",
3677                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3678         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3679         d.addCallback(self.failUnlessIsBarDotTxt)
3680         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3681         d.addCallback(self.failUnlessIsBarJSON)
3682         return d
3683
3684     def test_POST_relink_file_to_slash_fail(self):
3685         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3686                              "400 Bad Request",
3687                              "to_name= may not contain a slash",
3688                              self.POST, self.public_url + "/foo", t="relink",
3689                              from_name="bar.txt",
3690                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3691         d.addCallback(lambda res:
3692                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3693         d.addCallback(lambda res:
3694                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3695         d.addCallback(lambda ign:
3696                       self.shouldFail2(error.Error,
3697                                        "test_POST_rename_file_slash_fail2",
3698                                        "400 Bad Request",
3699                                        "from_name= may not contain a slash",
3700                                        self.POST, self.public_url + "/foo",
3701                                        t="relink",
3702                                        from_name="nope/bar.txt",
3703                                        to_name="fail.txt",
3704                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3705         return d
3706
3707     def test_POST_relink_file_explicitly_same_link(self):
3708         d = self.POST(self.public_url + "/foo", t="relink",
3709                       from_name="bar.txt",
3710                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3711         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3712         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3713         d.addCallback(self.failUnlessIsBarDotTxt)
3714         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3715         d.addCallback(self.failUnlessIsBarJSON)
3716         return d
3717
3718     def test_POST_relink_file_implicitly_same_link(self):
3719         d = self.POST(self.public_url + "/foo", t="relink",
3720                       from_name="bar.txt")
3721         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3722         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3723         d.addCallback(self.failUnlessIsBarDotTxt)
3724         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3725         d.addCallback(self.failUnlessIsBarJSON)
3726         return d
3727
3728     def test_POST_relink_file_same_dir(self):
3729         d = self.POST(self.public_url + "/foo", t="relink",
3730                       from_name="bar.txt",
3731                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3732         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3733         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3734         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3735         d.addCallback(self.failUnlessIsBarDotTxt)
3736         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3737         d.addCallback(self.failUnlessIsBarJSON)
3738         return d
3739
3740     def test_POST_relink_file_bad_replace(self):
3741         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3742                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3743                              self.POST,
3744                              self.public_url + "/foo", t="relink",
3745                              replace="boogabooga", from_name="bar.txt",
3746                              to_dir=self.public_root.get_uri() + "/foo/sub")
3747         return d
3748
3749     def test_POST_relink_file_multi_level(self):
3750         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3751         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3752                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3753         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3754         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3755         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3756         d.addCallback(self.failUnlessIsBarDotTxt)
3757         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3758         d.addCallback(self.failUnlessIsBarJSON)
3759         return d
3760
3761     def test_POST_relink_file_to_uri(self):
3762         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3763                       from_name="bar.txt", to_dir=self._sub_uri)
3764         d.addCallback(lambda res:
3765                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3766         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3767         d.addCallback(self.failUnlessIsBarDotTxt)
3768         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3769         d.addCallback(self.failUnlessIsBarJSON)
3770         return d
3771
3772     def test_POST_relink_file_to_nonexistent_dir(self):
3773         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3774                             "404 Not Found", "No such child: nopechucktesta",
3775                             self.POST, self.public_url + "/foo", t="relink",
3776                             from_name="bar.txt",
3777                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3778         return d
3779
3780     def test_POST_relink_file_into_file(self):
3781         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3782                              "400 Bad Request", "to_dir is not a directory",
3783                              self.POST, self.public_url + "/foo", t="relink",
3784                              from_name="bar.txt",
3785                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3786         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3787         d.addCallback(self.failUnlessIsBazDotTxt)
3788         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3789         d.addCallback(self.failUnlessIsBarDotTxt)
3790         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3791         d.addCallback(self.failUnlessIsBarJSON)
3792         return d
3793
3794     def test_POST_relink_file_to_bad_uri(self):
3795         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3796                               "400 Bad Request", "to_dir is not a directory",
3797                               self.POST, self.public_url + "/foo", t="relink",
3798                               from_name="bar.txt",
3799                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3800         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3801         d.addCallback(self.failUnlessIsBarDotTxt)
3802         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3803         d.addCallback(self.failUnlessIsBarJSON)
3804         return d
3805
3806     def test_POST_relink_dir(self):
3807         d = self.POST(self.public_url + "/foo", t="relink",
3808                       from_name="bar.txt",
3809                       to_dir=self.public_root.get_uri() + "/foo/empty")
3810         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3811                       t="relink", from_name="empty",
3812                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3813         d.addCallback(lambda res:
3814                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3815         d.addCallback(lambda res:
3816                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3817         d.addCallback(lambda res:
3818                       self._sub_node.get_child_at_path(u"empty"))
3819         d.addCallback(lambda node:
3820                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3821         d.addCallback(lambda res:
3822                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3823         d.addCallback(self.failUnlessIsBarDotTxt)
3824         return d
3825
3826     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3827         """ If target is not None then the redirection has to go to target.  If
3828         statuscode is not None then the redirection has to be accomplished with
3829         that HTTP status code."""
3830         if not isinstance(res, failure.Failure):
3831             to_where = (target is None) and "somewhere" or ("to " + target)
3832             self.fail("%s: we were expecting to get redirected %s, not get an"
3833                       " actual page: %s" % (which, to_where, res))
3834         res.trap(error.PageRedirect)
3835         if statuscode is not None:
3836             self.failUnlessReallyEqual(res.value.status, statuscode,
3837                                        "%s: not a redirect" % which)
3838         if target is not None:
3839             # the PageRedirect does not seem to capture the uri= query arg
3840             # properly, so we can't check for it.
3841             realtarget = self.webish_url + target
3842             self.failUnlessReallyEqual(res.value.location, realtarget,
3843                                        "%s: wrong target" % which)
3844         return res.value.location
3845
3846     def test_GET_URI_form(self):
3847         base = "/uri?uri=%s" % self._bar_txt_uri
3848         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3849         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3850         d = self.GET(base)
3851         d.addBoth(self.shouldRedirect, targetbase)
3852         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3853         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3854         d.addCallback(lambda res: self.GET(base+"&t=json"))
3855         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3856         d.addCallback(self.log, "about to get file by uri")
3857         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3858         d.addCallback(self.failUnlessIsBarDotTxt)
3859         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3860         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3861                                            followRedirect=True))
3862         d.addCallback(self.failUnlessIsFooJSON)
3863         d.addCallback(self.log, "got dir by uri")
3864
3865         return d
3866
3867     def test_GET_URI_form_bad(self):
3868         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3869                              "400 Bad Request", "GET /uri requires uri=",
3870                              self.GET, "/uri")
3871         return d
3872
3873     def test_GET_rename_form(self):
3874         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3875                      followRedirect=True)
3876         def _check(res):
3877             self.failUnlessIn('name="when_done" value="."', res)
3878             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3879             self.failUnlessIn(FAVICON_MARKUP, res)
3880         d.addCallback(_check)
3881         return d
3882
3883     def log(self, res, msg):
3884         #print "MSG: %s  RES: %s" % (msg, res)
3885         log.msg(msg)
3886         return res
3887
3888     def test_GET_URI_URL(self):
3889         base = "/uri/%s" % self._bar_txt_uri
3890         d = self.GET(base)
3891         d.addCallback(self.failUnlessIsBarDotTxt)
3892         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3893         d.addCallback(self.failUnlessIsBarDotTxt)
3894         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3895         d.addCallback(self.failUnlessIsBarDotTxt)
3896         return d
3897
3898     def test_GET_URI_URL_dir(self):
3899         base = "/uri/%s?t=json" % self._foo_uri
3900         d = self.GET(base)
3901         d.addCallback(self.failUnlessIsFooJSON)
3902         return d
3903
3904     def test_GET_URI_URL_missing(self):
3905         base = "/uri/%s" % self._bad_file_uri
3906         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3907                                  http.GONE, None, "NotEnoughSharesError",
3908                                  self.GET, base)
3909         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3910         # here? we must arrange for a download to fail after target.open()
3911         # has been called, and then inspect the response to see that it is
3912         # shorter than we expected.
3913         return d
3914
3915     def test_PUT_DIRURL_uri(self):
3916         d = self.s.create_dirnode()
3917         def _made_dir(dn):
3918             new_uri = dn.get_uri()
3919             # replace /foo with a new (empty) directory
3920             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3921             d.addCallback(lambda res:
3922                           self.failUnlessReallyEqual(res.strip(), new_uri))
3923             d.addCallback(lambda res:
3924                           self.failUnlessRWChildURIIs(self.public_root,
3925                                                       u"foo",
3926                                                       new_uri))
3927             return d
3928         d.addCallback(_made_dir)
3929         return d
3930
3931     def test_PUT_DIRURL_uri_noreplace(self):
3932         d = self.s.create_dirnode()
3933         def _made_dir(dn):
3934             new_uri = dn.get_uri()
3935             # replace /foo with a new (empty) directory, but ask that
3936             # replace=false, so it should fail
3937             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3938                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3939                                  self.PUT,
3940                                  self.public_url + "/foo?t=uri&replace=false",
3941                                  new_uri)
3942             d.addCallback(lambda res:
3943                           self.failUnlessRWChildURIIs(self.public_root,
3944                                                       u"foo",
3945                                                       self._foo_uri))
3946             return d
3947         d.addCallback(_made_dir)
3948         return d
3949
3950     def test_PUT_DIRURL_bad_t(self):
3951         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3952                              "400 Bad Request", "PUT to a directory",
3953                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3954         d.addCallback(lambda res:
3955                       self.failUnlessRWChildURIIs(self.public_root,
3956                                                   u"foo",
3957                                                   self._foo_uri))
3958         return d
3959
3960     def test_PUT_NEWFILEURL_uri(self):
3961         contents, n, new_uri = self.makefile(8)
3962         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3963         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3964         d.addCallback(lambda res:
3965                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3966                                                       contents))
3967         return d
3968
3969     def test_PUT_NEWFILEURL_mdmf(self):
3970         new_contents = self.NEWFILE_CONTENTS * 300000
3971         d = self.PUT(self.public_url + \
3972                      "/foo/mdmf.txt?format=mdmf",
3973                      new_contents)
3974         d.addCallback(lambda ignored:
3975             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3976         def _got_json(json):
3977             data = simplejson.loads(json)
3978             data = data[1]
3979             self.failUnlessIn("format", data)
3980             self.failUnlessEqual(data["format"], "MDMF")
3981             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3982             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3983         d.addCallback(_got_json)
3984         return d
3985
3986     def test_PUT_NEWFILEURL_sdmf(self):
3987         new_contents = self.NEWFILE_CONTENTS * 300000
3988         d = self.PUT(self.public_url + \
3989                      "/foo/sdmf.txt?format=sdmf",
3990                      new_contents)
3991         d.addCallback(lambda ignored:
3992             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3993         def _got_json(json):
3994             data = simplejson.loads(json)
3995             data = data[1]
3996             self.failUnlessIn("format", data)
3997             self.failUnlessEqual(data["format"], "SDMF")
3998         d.addCallback(_got_json)
3999         return d
4000
4001     def test_PUT_NEWFILEURL_bad_format(self):
4002         new_contents = self.NEWFILE_CONTENTS * 300000
4003         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4004                                     400, "Bad Request", "Unknown format: foo",
4005                                     self.PUT, self.public_url + \
4006                                     "/foo/foo.txt?format=foo",
4007                                     new_contents)
4008
4009     def test_PUT_NEWFILEURL_uri_replace(self):
4010         contents, n, new_uri = self.makefile(8)
4011         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4012         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4013         d.addCallback(lambda res:
4014                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4015                                                       contents))
4016         return d
4017
4018     def test_PUT_NEWFILEURL_uri_no_replace(self):
4019         contents, n, new_uri = self.makefile(8)
4020         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4021         d.addBoth(self.shouldFail, error.Error,
4022                   "PUT_NEWFILEURL_uri_no_replace",
4023                   "409 Conflict",
4024                   "There was already a child by that name, and you asked me "
4025                   "to not replace it")
4026         return d
4027
4028     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4029         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4030         d.addBoth(self.shouldFail, error.Error,
4031                   "POST_put_uri_unknown_bad",
4032                   "400 Bad Request",
4033                   "unknown cap in a write slot")
4034         return d
4035
4036     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4037         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4038         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4039                       u"put-future-ro.txt")
4040         return d
4041
4042     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4043         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4044         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4045                       u"put-future-imm.txt")
4046         return d
4047
4048     def test_PUT_NEWFILE_URI(self):
4049         file_contents = "New file contents here\n"
4050         d = self.PUT("/uri", file_contents)
4051         def _check(uri):
4052             assert isinstance(uri, str), uri
4053             self.failUnlessIn(uri, self.get_all_contents())
4054             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4055                                        file_contents)
4056             return self.GET("/uri/%s" % uri)
4057         d.addCallback(_check)
4058         def _check2(res):
4059             self.failUnlessReallyEqual(res, file_contents)
4060         d.addCallback(_check2)
4061         return d
4062
4063     def test_PUT_NEWFILE_URI_not_mutable(self):
4064         file_contents = "New file contents here\n"
4065         d = self.PUT("/uri?mutable=false", file_contents)
4066         def _check(uri):
4067             assert isinstance(uri, str), uri
4068             self.failUnlessIn(uri, self.get_all_contents())
4069             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4070                                        file_contents)
4071             return self.GET("/uri/%s" % uri)
4072         d.addCallback(_check)
4073         def _check2(res):
4074             self.failUnlessReallyEqual(res, file_contents)
4075         d.addCallback(_check2)
4076         return d
4077
4078     def test_PUT_NEWFILE_URI_only_PUT(self):
4079         d = self.PUT("/uri?t=bogus", "")
4080         d.addBoth(self.shouldFail, error.Error,
4081                   "PUT_NEWFILE_URI_only_PUT",
4082                   "400 Bad Request",
4083                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4084         return d
4085
4086     def test_PUT_NEWFILE_URI_mutable(self):
4087         file_contents = "New file contents here\n"
4088         d = self.PUT("/uri?mutable=true", file_contents)
4089         def _check1(filecap):
4090             filecap = filecap.strip()
4091             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4092             self.filecap = filecap
4093             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4094             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4095             n = self.s.create_node_from_uri(filecap)
4096             return n.download_best_version()
4097         d.addCallback(_check1)
4098         def _check2(data):
4099             self.failUnlessReallyEqual(data, file_contents)
4100             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4101         d.addCallback(_check2)
4102         def _check3(res):
4103             self.failUnlessReallyEqual(res, file_contents)
4104         d.addCallback(_check3)
4105         return d
4106
4107     def test_PUT_mkdir(self):
4108         d = self.PUT("/uri?t=mkdir", "")
4109         def _check(uri):
4110             n = self.s.create_node_from_uri(uri.strip())
4111             d2 = self.failUnlessNodeKeysAre(n, [])
4112             d2.addCallback(lambda res:
4113                            self.GET("/uri/%s?t=json" % uri))
4114             return d2
4115         d.addCallback(_check)
4116         d.addCallback(self.failUnlessIsEmptyJSON)
4117         return d
4118
4119     def test_PUT_mkdir_mdmf(self):
4120         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4121         def _got(res):
4122             u = uri.from_string(res)
4123             # Check that this is an MDMF writecap
4124             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4125         d.addCallback(_got)
4126         return d
4127
4128     def test_PUT_mkdir_sdmf(self):
4129         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4130         def _got(res):
4131             u = uri.from_string(res)
4132             self.failUnlessIsInstance(u, uri.DirectoryURI)
4133         d.addCallback(_got)
4134         return d
4135
4136     def test_PUT_mkdir_bad_format(self):
4137         return self.shouldHTTPError("PUT_mkdir_bad_format",
4138                                     400, "Bad Request", "Unknown format: foo",
4139                                     self.PUT, "/uri?t=mkdir&format=foo",
4140                                     "")
4141
4142     def test_POST_check(self):
4143         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4144         def _done(res):
4145             # this returns a string form of the results, which are probably
4146             # None since we're using fake filenodes.
4147             # TODO: verify that the check actually happened, by changing
4148             # FakeCHKFileNode to count how many times .check() has been
4149             # called.
4150             pass
4151         d.addCallback(_done)
4152         return d
4153
4154
4155     def test_PUT_update_at_offset(self):
4156         file_contents = "test file" * 100000 # about 900 KiB
4157         d = self.PUT("/uri?mutable=true", file_contents)
4158         def _then(filecap):
4159             self.filecap = filecap
4160             new_data = file_contents[:100]
4161             new = "replaced and so on"
4162             new_data += new
4163             new_data += file_contents[len(new_data):]
4164             assert len(new_data) == len(file_contents)
4165             self.new_data = new_data
4166         d.addCallback(_then)
4167         d.addCallback(lambda ignored:
4168             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4169                      "replaced and so on"))
4170         def _get_data(filecap):
4171             n = self.s.create_node_from_uri(filecap)
4172             return n.download_best_version()
4173         d.addCallback(_get_data)
4174         d.addCallback(lambda results:
4175             self.failUnlessEqual(results, self.new_data))
4176         # Now try appending things to the file
4177         d.addCallback(lambda ignored:
4178             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4179                      "puppies" * 100))
4180         d.addCallback(_get_data)
4181         d.addCallback(lambda results:
4182             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4183         # and try replacing the beginning of the file
4184         d.addCallback(lambda ignored:
4185             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4186         d.addCallback(_get_data)
4187         d.addCallback(lambda results:
4188             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4189         return d
4190
4191     def test_PUT_update_at_invalid_offset(self):
4192         file_contents = "test file" * 100000 # about 900 KiB
4193         d = self.PUT("/uri?mutable=true", file_contents)
4194         def _then(filecap):
4195             self.filecap = filecap
4196         d.addCallback(_then)
4197         # Negative offsets should cause an error.
4198         d.addCallback(lambda ignored:
4199             self.shouldHTTPError("PUT_update_at_invalid_offset",
4200                                  400, "Bad Request",
4201                                  "Invalid offset",
4202                                  self.PUT,
4203                                  "/uri/%s?offset=-1" % self.filecap,
4204                                  "foo"))
4205         return d
4206
4207     def test_PUT_update_at_offset_immutable(self):
4208         file_contents = "Test file" * 100000
4209         d = self.PUT("/uri", file_contents)
4210         def _then(filecap):
4211             self.filecap = filecap
4212         d.addCallback(_then)
4213         d.addCallback(lambda ignored:
4214             self.shouldHTTPError("PUT_update_at_offset_immutable",
4215                                  400, "Bad Request",
4216                                  "immutable",
4217                                  self.PUT,
4218                                  "/uri/%s?offset=50" % self.filecap,
4219                                  "foo"))
4220         return d
4221
4222
4223     def test_bad_method(self):
4224         url = self.webish_url + self.public_url + "/foo/bar.txt"
4225         d = self.shouldHTTPError("bad_method",
4226                                  501, "Not Implemented",
4227                                  "I don't know how to treat a BOGUS request.",
4228                                  client.getPage, url, method="BOGUS")
4229         return d
4230
4231     def test_short_url(self):
4232         url = self.webish_url + "/uri"
4233         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4234                                  "I don't know how to treat a DELETE request.",
4235                                  client.getPage, url, method="DELETE")
4236         return d
4237
4238     def test_ophandle_bad(self):
4239         url = self.webish_url + "/operations/bogus?t=status"
4240         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4241                                  "unknown/expired handle 'bogus'",
4242                                  client.getPage, url)
4243         return d
4244
4245     def test_ophandle_cancel(self):
4246         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4247                       followRedirect=True)
4248         d.addCallback(lambda ignored:
4249                       self.GET("/operations/128?t=status&output=JSON"))
4250         def _check1(res):
4251             data = simplejson.loads(res)
4252             self.failUnless("finished" in data, res)
4253             monitor = self.ws.root.child_operations.handles["128"][0]
4254             d = self.POST("/operations/128?t=cancel&output=JSON")
4255             def _check2(res):
4256                 data = simplejson.loads(res)
4257                 self.failUnless("finished" in data, res)
4258                 # t=cancel causes the handle to be forgotten
4259                 self.failUnless(monitor.is_cancelled())
4260             d.addCallback(_check2)
4261             return d
4262         d.addCallback(_check1)
4263         d.addCallback(lambda ignored:
4264                       self.shouldHTTPError("ophandle_cancel",
4265                                            404, "404 Not Found",
4266                                            "unknown/expired handle '128'",
4267                                            self.GET,
4268                                            "/operations/128?t=status&output=JSON"))
4269         return d
4270
4271     def test_ophandle_retainfor(self):
4272         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4273                       followRedirect=True)
4274         d.addCallback(lambda ignored:
4275                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4276         def _check1(res):
4277             data = simplejson.loads(res)
4278             self.failUnless("finished" in data, res)
4279         d.addCallback(_check1)
4280         # the retain-for=0 will cause the handle to be expired very soon
4281         d.addCallback(lambda ign:
4282             self.clock.advance(2.0))
4283         d.addCallback(lambda ignored:
4284                       self.shouldHTTPError("ophandle_retainfor",
4285                                            404, "404 Not Found",
4286                                            "unknown/expired handle '129'",
4287                                            self.GET,
4288                                            "/operations/129?t=status&output=JSON"))
4289         return d
4290
4291     def test_ophandle_release_after_complete(self):
4292         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4293                       followRedirect=True)
4294         d.addCallback(self.wait_for_operation, "130")
4295         d.addCallback(lambda ignored:
4296                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4297         # the release-after-complete=true will cause the handle to be expired
4298         d.addCallback(lambda ignored:
4299                       self.shouldHTTPError("ophandle_release_after_complete",
4300                                            404, "404 Not Found",
4301                                            "unknown/expired handle '130'",
4302                                            self.GET,
4303                                            "/operations/130?t=status&output=JSON"))
4304         return d
4305
4306     def test_uncollected_ophandle_expiration(self):
4307         # uncollected ophandles should expire after 4 days
4308         def _make_uncollected_ophandle(ophandle):
4309             d = self.POST(self.public_url +
4310                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4311                           followRedirect=False)
4312             # When we start the operation, the webapi server will want
4313             # to redirect us to the page for the ophandle, so we get
4314             # confirmation that the operation has started. If the
4315             # manifest operation has finished by the time we get there,
4316             # following that redirect (by setting followRedirect=True
4317             # above) has the side effect of collecting the ophandle that
4318             # we've just created, which means that we can't use the
4319             # ophandle to test the uncollected timeout anymore. So,
4320             # instead, catch the 302 here and don't follow it.
4321             d.addBoth(self.should302, "uncollected_ophandle_creation")
4322             return d
4323         # Create an ophandle, don't collect it, then advance the clock by
4324         # 4 days - 1 second and make sure that the ophandle is still there.
4325         d = _make_uncollected_ophandle(131)
4326         d.addCallback(lambda ign:
4327             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4328         d.addCallback(lambda ign:
4329             self.GET("/operations/131?t=status&output=JSON"))
4330         def _check1(res):
4331             data = simplejson.loads(res)
4332             self.failUnless("finished" in data, res)
4333         d.addCallback(_check1)
4334         # Create an ophandle, don't collect it, then try to collect it
4335         # after 4 days. It should be gone.
4336         d.addCallback(lambda ign:
4337             _make_uncollected_ophandle(132))
4338         d.addCallback(lambda ign:
4339             self.clock.advance(96*60*60))
4340         d.addCallback(lambda ign:
4341             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4342                                  404, "404 Not Found",
4343                                  "unknown/expired handle '132'",
4344                                  self.GET,
4345                                  "/operations/132?t=status&output=JSON"))
4346         return d
4347
4348     def test_collected_ophandle_expiration(self):
4349         # collected ophandles should expire after 1 day
4350         def _make_collected_ophandle(ophandle):
4351             d = self.POST(self.public_url +
4352                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4353                           followRedirect=True)
4354             # By following the initial redirect, we collect the ophandle
4355             # we've just created.
4356             return d
4357         # Create a collected ophandle, then collect it after 23 hours
4358         # and 59 seconds to make sure that it is still there.
4359         d = _make_collected_ophandle(133)
4360         d.addCallback(lambda ign:
4361             self.clock.advance((24*60*60) - 1))
4362         d.addCallback(lambda ign:
4363             self.GET("/operations/133?t=status&output=JSON"))
4364         def _check1(res):
4365             data = simplejson.loads(res)
4366             self.failUnless("finished" in data, res)
4367         d.addCallback(_check1)
4368         # Create another uncollected ophandle, then try to collect it
4369         # after 24 hours to make sure that it is gone.
4370         d.addCallback(lambda ign:
4371             _make_collected_ophandle(134))
4372         d.addCallback(lambda ign:
4373             self.clock.advance(24*60*60))
4374         d.addCallback(lambda ign:
4375             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4376                                  404, "404 Not Found",
4377                                  "unknown/expired handle '134'",
4378                                  self.GET,
4379                                  "/operations/134?t=status&output=JSON"))
4380         return d
4381
4382     def test_incident(self):
4383         d = self.POST("/report_incident", details="eek")
4384         def _done(res):
4385             self.failIfIn("<html>", res)
4386             self.failUnlessIn("An incident report has been saved", res)
4387         d.addCallback(_done)
4388         return d
4389
4390     def test_static(self):
4391         webdir = os.path.join(self.staticdir, "subdir")
4392         fileutil.make_dirs(webdir)
4393         f = open(os.path.join(webdir, "hello.txt"), "wb")
4394         f.write("hello")
4395         f.close()
4396
4397         d = self.GET("/static/subdir/hello.txt")
4398         def _check(res):
4399             self.failUnlessReallyEqual(res, "hello")
4400         d.addCallback(_check)
4401         return d
4402
4403
4404 class IntroducerWeb(unittest.TestCase):
4405     def setUp(self):
4406         self.node = None
4407
4408     def tearDown(self):
4409         d = defer.succeed(None)
4410         if self.node:
4411             d.addCallback(lambda ign: self.node.stopService())
4412         d.addCallback(flushEventualQueue)
4413         return d
4414
4415     def test_welcome(self):
4416         basedir = "web.IntroducerWeb.test_welcome"
4417         os.mkdir(basedir)
4418         cfg = "\n".join(["[node]",
4419                          "tub.location = 127.0.0.1:1",
4420                          "web.port = tcp:0",
4421                          ]) + "\n"
4422         fileutil.write(os.path.join(basedir, "tahoe.cfg"), cfg)
4423         self.node = IntroducerNode(basedir)
4424         self.ws = self.node.getServiceNamed("webish")
4425
4426         d = fireEventually(None)
4427         d.addCallback(lambda ign: self.node.startService())
4428         d.addCallback(lambda ign: self.node.when_tub_ready())
4429
4430         d.addCallback(lambda ign: self.GET("/"))
4431         def _check(res):
4432             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4433             self.failUnlessIn(FAVICON_MARKUP, res)
4434             self.failUnlessIn('Page rendered at', res)
4435             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4436         d.addCallback(_check)
4437         return d
4438
4439     def GET(self, urlpath, followRedirect=False, return_response=False,
4440             **kwargs):
4441         # if return_response=True, this fires with (data, statuscode,
4442         # respheaders) instead of just data.
4443         assert not isinstance(urlpath, unicode)
4444         url = self.ws.getURL().rstrip('/') + urlpath
4445         factory = HTTPClientGETFactory(url, method="GET",
4446                                        followRedirect=followRedirect, **kwargs)
4447         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4448         d = factory.deferred
4449         def _got_data(data):
4450             return (data, factory.status, factory.response_headers)
4451         if return_response:
4452             d.addCallback(_got_data)
4453         return factory.deferred
4454
4455
4456 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4457     def test_load_file(self):
4458         # This will raise an exception unless a well-formed XML file is found under that name.
4459         common.getxmlfile('directory.xhtml').load()
4460
4461     def test_parse_replace_arg(self):
4462         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4463         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4464         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4465                                    "only-files")
4466         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4467
4468     def test_abbreviate_time(self):
4469         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4470         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4471         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4472         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4473         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4474         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4475
4476     def test_compute_rate(self):
4477         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4478         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4479         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4480         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4481         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4482         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4483         self.shouldFail(AssertionError, "test_compute_rate", "",
4484                         common.compute_rate, -100, 10)
4485         self.shouldFail(AssertionError, "test_compute_rate", "",
4486                         common.compute_rate, 100, -10)
4487
4488         # Sanity check
4489         rate = common.compute_rate(10*1000*1000, 1)
4490         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4491
4492     def test_abbreviate_rate(self):
4493         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4494         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4495         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4496         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4497
4498     def test_abbreviate_size(self):
4499         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4500         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4501         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4502         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4503         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4504
4505     def test_plural(self):
4506         def convert(s):
4507             return "%d second%s" % (s, status.plural(s))
4508         self.failUnlessReallyEqual(convert(0), "0 seconds")
4509         self.failUnlessReallyEqual(convert(1), "1 second")
4510         self.failUnlessReallyEqual(convert(2), "2 seconds")
4511         def convert2(s):
4512             return "has share%s: %s" % (status.plural(s), ",".join(s))
4513         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4514         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4515         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4516
4517
4518 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4519
4520     def CHECK(self, ign, which, args, clientnum=0):
4521         fileurl = self.fileurls[which]
4522         url = fileurl + "?" + args
4523         return self.GET(url, method="POST", clientnum=clientnum)
4524
4525     def test_filecheck(self):
4526         self.basedir = "web/Grid/filecheck"
4527         self.set_up_grid()
4528         c0 = self.g.clients[0]
4529         self.uris = {}
4530         DATA = "data" * 100
4531         d = c0.upload(upload.Data(DATA, convergence=""))
4532         def _stash_uri(ur, which):
4533             self.uris[which] = ur.get_uri()
4534         d.addCallback(_stash_uri, "good")
4535         d.addCallback(lambda ign:
4536                       c0.upload(upload.Data(DATA+"1", convergence="")))
4537         d.addCallback(_stash_uri, "sick")
4538         d.addCallback(lambda ign:
4539                       c0.upload(upload.Data(DATA+"2", convergence="")))
4540         d.addCallback(_stash_uri, "dead")
4541         def _stash_mutable_uri(n, which):
4542             self.uris[which] = n.get_uri()
4543             assert isinstance(self.uris[which], str)
4544         d.addCallback(lambda ign:
4545             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4546         d.addCallback(_stash_mutable_uri, "corrupt")
4547         d.addCallback(lambda ign:
4548                       c0.upload(upload.Data("literal", convergence="")))
4549         d.addCallback(_stash_uri, "small")
4550         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4551         d.addCallback(_stash_mutable_uri, "smalldir")
4552
4553         def _compute_fileurls(ignored):
4554             self.fileurls = {}
4555             for which in self.uris:
4556                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4557         d.addCallback(_compute_fileurls)
4558
4559         def _clobber_shares(ignored):
4560             good_shares = self.find_uri_shares(self.uris["good"])
4561             self.failUnlessReallyEqual(len(good_shares), 10)
4562             sick_shares = self.find_uri_shares(self.uris["sick"])
4563             os.unlink(sick_shares[0][2])
4564             dead_shares = self.find_uri_shares(self.uris["dead"])
4565             for i in range(1, 10):
4566                 os.unlink(dead_shares[i][2])
4567             c_shares = self.find_uri_shares(self.uris["corrupt"])
4568             cso = CorruptShareOptions()
4569             cso.stdout = StringIO()
4570             cso.parseOptions([c_shares[0][2]])
4571             corrupt_share(cso)
4572         d.addCallback(_clobber_shares)
4573
4574         d.addCallback(self.CHECK, "good", "t=check")
4575         def _got_html_good(res):
4576             self.failUnlessIn("Healthy", res)
4577             self.failIfIn("Not Healthy", res)
4578             self.failUnlessIn(FAVICON_MARKUP, res)
4579         d.addCallback(_got_html_good)
4580         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4581         def _got_html_good_return_to(res):
4582             self.failUnlessIn("Healthy", res)
4583             self.failIfIn("Not Healthy", res)
4584             self.failUnlessIn('<a href="somewhere">Return to file', res)
4585         d.addCallback(_got_html_good_return_to)
4586         d.addCallback(self.CHECK, "good", "t=check&output=json")
4587         def _got_json_good(res):
4588             r = simplejson.loads(res)
4589             self.failUnlessEqual(r["summary"], "Healthy")
4590             self.failUnless(r["results"]["healthy"])
4591             self.failIfIn("needs-rebalancing", r["results"])
4592             self.failUnless(r["results"]["recoverable"])
4593         d.addCallback(_got_json_good)
4594
4595         d.addCallback(self.CHECK, "small", "t=check")
4596         def _got_html_small(res):
4597             self.failUnlessIn("Literal files are always healthy", res)
4598             self.failIfIn("Not Healthy", res)
4599         d.addCallback(_got_html_small)
4600         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4601         def _got_html_small_return_to(res):
4602             self.failUnlessIn("Literal files are always healthy", res)
4603             self.failIfIn("Not Healthy", res)
4604             self.failUnlessIn('<a href="somewhere">Return to file', res)
4605         d.addCallback(_got_html_small_return_to)
4606         d.addCallback(self.CHECK, "small", "t=check&output=json")
4607         def _got_json_small(res):
4608             r = simplejson.loads(res)
4609             self.failUnlessEqual(r["storage-index"], "")
4610             self.failUnless(r["results"]["healthy"])
4611         d.addCallback(_got_json_small)
4612
4613         d.addCallback(self.CHECK, "smalldir", "t=check")
4614         def _got_html_smalldir(res):
4615             self.failUnlessIn("Literal files are always healthy", res)
4616             self.failIfIn("Not Healthy", res)
4617         d.addCallback(_got_html_smalldir)
4618         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4619         def _got_json_smalldir(res):
4620             r = simplejson.loads(res)
4621             self.failUnlessEqual(r["storage-index"], "")
4622             self.failUnless(r["results"]["healthy"])
4623         d.addCallback(_got_json_smalldir)
4624
4625         d.addCallback(self.CHECK, "sick", "t=check")
4626         def _got_html_sick(res):
4627             self.failUnlessIn("Not Healthy", res)
4628         d.addCallback(_got_html_sick)
4629         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4630         def _got_json_sick(res):
4631             r = simplejson.loads(res)
4632             self.failUnlessEqual(r["summary"],
4633                                  "Not Healthy: 9 shares (enc 3-of-10)")
4634             self.failIf(r["results"]["healthy"])
4635             self.failUnless(r["results"]["recoverable"])
4636             self.failIfIn("needs-rebalancing", r["results"])
4637         d.addCallback(_got_json_sick)
4638
4639         d.addCallback(self.CHECK, "dead", "t=check")
4640         def _got_html_dead(res):
4641             self.failUnlessIn("Not Healthy", res)
4642         d.addCallback(_got_html_dead)
4643         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4644         def _got_json_dead(res):
4645             r = simplejson.loads(res)
4646             self.failUnlessEqual(r["summary"],
4647                                  "Not Healthy: 1 shares (enc 3-of-10)")
4648             self.failIf(r["results"]["healthy"])
4649             self.failIf(r["results"]["recoverable"])
4650             self.failIfIn("needs-rebalancing", r["results"])
4651         d.addCallback(_got_json_dead)
4652
4653         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4654         def _got_html_corrupt(res):
4655             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4656         d.addCallback(_got_html_corrupt)
4657         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4658         def _got_json_corrupt(res):
4659             r = simplejson.loads(res)
4660             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4661             self.failIf(r["results"]["healthy"])
4662             self.failUnless(r["results"]["recoverable"])
4663             self.failIfIn("needs-rebalancing", r["results"])
4664             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4665             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4666             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4667         d.addCallback(_got_json_corrupt)
4668
4669         d.addErrback(self.explain_web_error)
4670         return d
4671
4672     def test_repair_html(self):
4673         self.basedir = "web/Grid/repair_html"
4674         self.set_up_grid()
4675         c0 = self.g.clients[0]
4676         self.uris = {}
4677         DATA = "data" * 100
4678         d = c0.upload(upload.Data(DATA, convergence=""))
4679         def _stash_uri(ur, which):
4680             self.uris[which] = ur.get_uri()
4681         d.addCallback(_stash_uri, "good")
4682         d.addCallback(lambda ign:
4683                       c0.upload(upload.Data(DATA+"1", convergence="")))
4684         d.addCallback(_stash_uri, "sick")
4685         d.addCallback(lambda ign:
4686                       c0.upload(upload.Data(DATA+"2", convergence="")))
4687         d.addCallback(_stash_uri, "dead")
4688         def _stash_mutable_uri(n, which):
4689             self.uris[which] = n.get_uri()
4690             assert isinstance(self.uris[which], str)
4691         d.addCallback(lambda ign:
4692             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4693         d.addCallback(_stash_mutable_uri, "corrupt")
4694
4695         def _compute_fileurls(ignored):
4696             self.fileurls = {}
4697             for which in self.uris:
4698                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4699         d.addCallback(_compute_fileurls)
4700
4701         def _clobber_shares(ignored):
4702             good_shares = self.find_uri_shares(self.uris["good"])
4703             self.failUnlessReallyEqual(len(good_shares), 10)
4704             sick_shares = self.find_uri_shares(self.uris["sick"])
4705             os.unlink(sick_shares[0][2])
4706             dead_shares = self.find_uri_shares(self.uris["dead"])
4707             for i in range(1, 10):
4708                 os.unlink(dead_shares[i][2])
4709             c_shares = self.find_uri_shares(self.uris["corrupt"])
4710             cso = CorruptShareOptions()
4711             cso.stdout = StringIO()
4712             cso.parseOptions([c_shares[0][2]])
4713             corrupt_share(cso)
4714         d.addCallback(_clobber_shares)
4715
4716         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4717         def _got_html_good(res):
4718             self.failUnlessIn("Healthy", res)
4719             self.failIfIn("Not Healthy", res)
4720             self.failUnlessIn("No repair necessary", res)
4721             self.failUnlessIn(FAVICON_MARKUP, res)
4722         d.addCallback(_got_html_good)
4723
4724         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4725         def _got_html_sick(res):
4726             self.failUnlessIn("Healthy : healthy", res)
4727             self.failIfIn("Not Healthy", res)
4728             self.failUnlessIn("Repair successful", res)
4729         d.addCallback(_got_html_sick)
4730
4731         # repair of a dead file will fail, of course, but it isn't yet
4732         # clear how this should be reported. Right now it shows up as
4733         # a "410 Gone".
4734         #
4735         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4736         #def _got_html_dead(res):
4737         #    print res
4738         #    self.failUnlessIn("Healthy : healthy", res)
4739         #    self.failIfIn("Not Healthy", res)
4740         #    self.failUnlessIn("No repair necessary", res)
4741         #d.addCallback(_got_html_dead)
4742
4743         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4744         def _got_html_corrupt(res):
4745             self.failUnlessIn("Healthy : Healthy", res)
4746             self.failIfIn("Not Healthy", res)
4747             self.failUnlessIn("Repair successful", res)
4748         d.addCallback(_got_html_corrupt)
4749
4750         d.addErrback(self.explain_web_error)
4751         return d
4752
4753     def test_repair_json(self):
4754         self.basedir = "web/Grid/repair_json"
4755         self.set_up_grid()
4756         c0 = self.g.clients[0]
4757         self.uris = {}
4758         DATA = "data" * 100
4759         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4760         def _stash_uri(ur, which):
4761             self.uris[which] = ur.get_uri()
4762         d.addCallback(_stash_uri, "sick")
4763
4764         def _compute_fileurls(ignored):
4765             self.fileurls = {}
4766             for which in self.uris:
4767                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4768         d.addCallback(_compute_fileurls)
4769
4770         def _clobber_shares(ignored):
4771             sick_shares = self.find_uri_shares(self.uris["sick"])
4772             os.unlink(sick_shares[0][2])
4773         d.addCallback(_clobber_shares)
4774
4775         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4776         def _got_json_sick(res):
4777             r = simplejson.loads(res)
4778             self.failUnlessReallyEqual(r["repair-attempted"], True)
4779             self.failUnlessReallyEqual(r["repair-successful"], True)
4780             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4781                                  "Not Healthy: 9 shares (enc 3-of-10)")
4782             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4783             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4784             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4785         d.addCallback(_got_json_sick)
4786
4787         d.addErrback(self.explain_web_error)
4788         return d
4789
4790     def test_unknown(self, immutable=False):
4791         self.basedir = "web/Grid/unknown"
4792         if immutable:
4793             self.basedir = "web/Grid/unknown-immutable"
4794
4795         self.set_up_grid()
4796         c0 = self.g.clients[0]
4797         self.uris = {}
4798         self.fileurls = {}
4799
4800         # the future cap format may contain slashes, which must be tolerated
4801         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4802                                                            safe="")
4803
4804         if immutable:
4805             name = u"future-imm"
4806             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4807             d = c0.create_immutable_dirnode({name: (future_node, {})})
4808         else:
4809             name = u"future"
4810             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4811             d = c0.create_dirnode()
4812
4813         def _stash_root_and_create_file(n):
4814             self.rootnode = n
4815             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4816             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4817             if not immutable:
4818                 return self.rootnode.set_node(name, future_node)
4819         d.addCallback(_stash_root_and_create_file)
4820
4821         # make sure directory listing tolerates unknown nodes
4822         d.addCallback(lambda ign: self.GET(self.rooturl))
4823         def _check_directory_html(res, expected_type_suffix):
4824             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4825                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4826                                  re.DOTALL)
4827             self.failUnless(re.search(pattern, res), res)
4828             # find the More Info link for name, should be relative
4829             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4830             info_url = mo.group(1)
4831             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4832         if immutable:
4833             d.addCallback(_check_directory_html, "-IMM")
4834         else:
4835             d.addCallback(_check_directory_html, "")
4836
4837         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4838         def _check_directory_json(res, expect_rw_uri):
4839             data = simplejson.loads(res)
4840             self.failUnlessEqual(data[0], "dirnode")
4841             f = data[1]["children"][name]
4842             self.failUnlessEqual(f[0], "unknown")
4843             if expect_rw_uri:
4844                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4845             else:
4846                 self.failIfIn("rw_uri", f[1])
4847             if immutable:
4848                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4849             else:
4850                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4851             self.failUnlessIn("metadata", f[1])
4852         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4853
4854         def _check_info(res, expect_rw_uri, expect_ro_uri):
4855             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4856             if expect_rw_uri:
4857                 self.failUnlessIn(unknown_rwcap, res)
4858             if expect_ro_uri:
4859                 if immutable:
4860                     self.failUnlessIn(unknown_immcap, res)
4861                 else:
4862                     self.failUnlessIn(unknown_rocap, res)
4863             else:
4864                 self.failIfIn(unknown_rocap, res)
4865             self.failIfIn("Raw data as", res)
4866             self.failIfIn("Directory writecap", res)
4867             self.failIfIn("Checker Operations", res)
4868             self.failIfIn("Mutable File Operations", res)
4869             self.failIfIn("Directory Operations", res)
4870
4871         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4872         # why they fail. Possibly related to ticket #922.
4873
4874         d.addCallback(lambda ign: self.GET(expected_info_url))
4875         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4876         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4877         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4878
4879         def _check_json(res, expect_rw_uri):
4880             data = simplejson.loads(res)
4881             self.failUnlessEqual(data[0], "unknown")
4882             if expect_rw_uri:
4883                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4884             else:
4885                 self.failIfIn("rw_uri", data[1])
4886
4887             if immutable:
4888                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4889                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4890             elif expect_rw_uri:
4891                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4892                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4893             else:
4894                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4895                 self.failIfIn("mutable", data[1])
4896
4897             # TODO: check metadata contents
4898             self.failUnlessIn("metadata", data[1])
4899
4900         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4901         d.addCallback(_check_json, expect_rw_uri=not immutable)
4902
4903         # and make sure that a read-only version of the directory can be
4904         # rendered too. This version will not have unknown_rwcap, whether
4905         # or not future_node was immutable.
4906         d.addCallback(lambda ign: self.GET(self.rourl))
4907         if immutable:
4908             d.addCallback(_check_directory_html, "-IMM")
4909         else:
4910             d.addCallback(_check_directory_html, "-RO")
4911
4912         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4913         d.addCallback(_check_directory_json, expect_rw_uri=False)
4914
4915         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4916         d.addCallback(_check_json, expect_rw_uri=False)
4917
4918         # TODO: check that getting t=info from the Info link in the ro directory
4919         # works, and does not include the writecap URI.
4920         return d
4921
4922     def test_immutable_unknown(self):
4923         return self.test_unknown(immutable=True)
4924
4925     def test_mutant_dirnodes_are_omitted(self):
4926         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4927
4928         self.set_up_grid()
4929         c = self.g.clients[0]
4930         nm = c.nodemaker
4931         self.uris = {}
4932         self.fileurls = {}
4933
4934         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4935         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4936         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4937
4938         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4939         # test the dirnode and web layers separately.
4940
4941         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4942         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4943         # When the directory is read, the mutants should be silently disposed of, leaving
4944         # their lonely sibling.
4945         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4946         # because immutable directories don't have a writecap and therefore that field
4947         # isn't (and can't be) decrypted.
4948         # TODO: The field still exists in the netstring. Technically we should check what
4949         # happens if something is put there (_unpack_contents should raise ValueError),
4950         # but that can wait.
4951
4952         lonely_child = nm.create_from_cap(lonely_uri)
4953         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4954         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4955
4956         def _by_hook_or_by_crook():
4957             return True
4958         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4959             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4960
4961         mutant_write_in_ro_child.get_write_uri    = lambda: None
4962         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4963
4964         kids = {u"lonely":      (lonely_child, {}),
4965                 u"ro":          (mutant_ro_child, {}),
4966                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4967                 }
4968         d = c.create_immutable_dirnode(kids)
4969
4970         def _created(dn):
4971             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4972             self.failIf(dn.is_mutable())
4973             self.failUnless(dn.is_readonly())
4974             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4975             self.failIf(hasattr(dn._node, 'get_writekey'))
4976             rep = str(dn)
4977             self.failUnlessIn("RO-IMM", rep)
4978             cap = dn.get_cap()
4979             self.failUnlessIn("CHK", cap.to_string())
4980             self.cap = cap
4981             self.rootnode = dn
4982             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4983             return download_to_data(dn._node)
4984         d.addCallback(_created)
4985
4986         def _check_data(data):
4987             # Decode the netstring representation of the directory to check that all children
4988             # are present. This is a bit of an abstraction violation, but there's not really
4989             # any other way to do it given that the real DirectoryNode._unpack_contents would
4990             # strip the mutant children out (which is what we're trying to test, later).
4991             position = 0
4992             numkids = 0
4993             while position < len(data):
4994                 entries, position = split_netstring(data, 1, position)
4995                 entry = entries[0]
4996                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4997                 name = name_utf8.decode("utf-8")
4998                 self.failUnlessEqual(rwcapdata, "")
4999                 self.failUnlessIn(name, kids)
5000                 (expected_child, ign) = kids[name]
5001                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
5002                 numkids += 1
5003
5004             self.failUnlessReallyEqual(numkids, 3)
5005             return self.rootnode.list()
5006         d.addCallback(_check_data)
5007
5008         # Now when we use the real directory listing code, the mutants should be absent.
5009         def _check_kids(children):
5010             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5011             lonely_node, lonely_metadata = children[u"lonely"]
5012
5013             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5014             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5015         d.addCallback(_check_kids)
5016
5017         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5018         d.addCallback(lambda n: n.list())
5019         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5020
5021         # Make sure the lonely child can be listed in HTML...
5022         d.addCallback(lambda ign: self.GET(self.rooturl))
5023         def _check_html(res):
5024             self.failIfIn("URI:SSK", res)
5025             get_lonely = "".join([r'<td>FILE</td>',
5026                                   r'\s+<td>',
5027                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5028                                   r'</td>',
5029                                   r'\s+<td align="right">%d</td>' % len("one"),
5030                                   ])
5031             self.failUnless(re.search(get_lonely, res), res)
5032
5033             # find the More Info link for name, should be relative
5034             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5035             info_url = mo.group(1)
5036             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5037         d.addCallback(_check_html)
5038
5039         # ... and in JSON.
5040         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5041         def _check_json(res):
5042             data = simplejson.loads(res)
5043             self.failUnlessEqual(data[0], "dirnode")
5044             listed_children = data[1]["children"]
5045             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5046             ll_type, ll_data = listed_children[u"lonely"]
5047             self.failUnlessEqual(ll_type, "filenode")
5048             self.failIfIn("rw_uri", ll_data)
5049             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5050         d.addCallback(_check_json)
5051         return d
5052
5053     def test_deep_check(self):
5054         self.basedir = "web/Grid/deep_check"
5055         self.set_up_grid()
5056         c0 = self.g.clients[0]
5057         self.uris = {}
5058         self.fileurls = {}
5059         DATA = "data" * 100
5060         d = c0.create_dirnode()
5061         def _stash_root_and_create_file(n):
5062             self.rootnode = n
5063             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5064             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5065         d.addCallback(_stash_root_and_create_file)
5066         def _stash_uri(fn, which):
5067             self.uris[which] = fn.get_uri()
5068             return fn
5069         d.addCallback(_stash_uri, "good")
5070         d.addCallback(lambda ign:
5071                       self.rootnode.add_file(u"small",
5072                                              upload.Data("literal",
5073                                                         convergence="")))
5074         d.addCallback(_stash_uri, "small")
5075         d.addCallback(lambda ign:
5076                       self.rootnode.add_file(u"sick",
5077                                              upload.Data(DATA+"1",
5078                                                         convergence="")))
5079         d.addCallback(_stash_uri, "sick")
5080
5081         # this tests that deep-check and stream-manifest will ignore
5082         # UnknownNode instances. Hopefully this will also cover deep-stats.
5083         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5084         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5085
5086         def _clobber_shares(ignored):
5087             self.delete_shares_numbered(self.uris["sick"], [0,1])
5088         d.addCallback(_clobber_shares)
5089
5090         # root
5091         # root/good
5092         # root/small
5093         # root/sick
5094         # root/future
5095
5096         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5097         def _done(res):
5098             try:
5099                 units = [simplejson.loads(line)
5100                          for line in res.splitlines()
5101                          if line]
5102             except ValueError:
5103                 print "response is:", res
5104                 print "undecodeable line was '%s'" % line
5105                 raise
5106             self.failUnlessReallyEqual(len(units), 5+1)
5107             # should be parent-first
5108             u0 = units[0]
5109             self.failUnlessEqual(u0["path"], [])
5110             self.failUnlessEqual(u0["type"], "directory")
5111             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5112             u0cr = u0["check-results"]
5113             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5114             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5115
5116             ugood = [u for u in units
5117                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5118             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5119             ugoodcr = ugood["check-results"]
5120             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5121             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5122
5123             stats = units[-1]
5124             self.failUnlessEqual(stats["type"], "stats")
5125             s = stats["stats"]
5126             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5127             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5128             self.failUnlessReallyEqual(s["count-directories"], 1)
5129             self.failUnlessReallyEqual(s["count-unknown"], 1)
5130         d.addCallback(_done)
5131
5132         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5133         def _check_manifest(res):
5134             self.failUnless(res.endswith("\n"))
5135             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5136             self.failUnlessReallyEqual(len(units), 5+1)
5137             self.failUnlessEqual(units[-1]["type"], "stats")
5138             first = units[0]
5139             self.failUnlessEqual(first["path"], [])
5140             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5141             self.failUnlessEqual(first["type"], "directory")
5142             stats = units[-1]["stats"]
5143             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5144             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5145             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5146             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5147             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5148         d.addCallback(_check_manifest)
5149
5150         # now add root/subdir and root/subdir/grandchild, then make subdir
5151         # unrecoverable, then see what happens
5152
5153         d.addCallback(lambda ign:
5154                       self.rootnode.create_subdirectory(u"subdir"))
5155         d.addCallback(_stash_uri, "subdir")
5156         d.addCallback(lambda subdir_node:
5157                       subdir_node.add_file(u"grandchild",
5158                                            upload.Data(DATA+"2",
5159                                                        convergence="")))
5160         d.addCallback(_stash_uri, "grandchild")
5161
5162         d.addCallback(lambda ign:
5163                       self.delete_shares_numbered(self.uris["subdir"],
5164                                                   range(1, 10)))
5165
5166         # root
5167         # root/good
5168         # root/small
5169         # root/sick
5170         # root/future
5171         # root/subdir [unrecoverable]
5172         # root/subdir/grandchild
5173
5174         # how should a streaming-JSON API indicate fatal error?
5175         # answer: emit ERROR: instead of a JSON string
5176
5177         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5178         def _check_broken_manifest(res):
5179             lines = res.splitlines()
5180             error_lines = [i
5181                            for (i,line) in enumerate(lines)
5182                            if line.startswith("ERROR:")]
5183             if not error_lines:
5184                 self.fail("no ERROR: in output: %s" % (res,))
5185             first_error = error_lines[0]
5186             error_line = lines[first_error]
5187             error_msg = lines[first_error+1:]
5188             error_msg_s = "\n".join(error_msg) + "\n"
5189             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5190                               error_line)
5191             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5192             units = [simplejson.loads(line) for line in lines[:first_error]]
5193             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5194             last_unit = units[-1]
5195             self.failUnlessEqual(last_unit["path"], ["subdir"])
5196         d.addCallback(_check_broken_manifest)
5197
5198         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5199         def _check_broken_deepcheck(res):
5200             lines = res.splitlines()
5201             error_lines = [i
5202                            for (i,line) in enumerate(lines)
5203                            if line.startswith("ERROR:")]
5204             if not error_lines:
5205                 self.fail("no ERROR: in output: %s" % (res,))
5206             first_error = error_lines[0]
5207             error_line = lines[first_error]
5208             error_msg = lines[first_error+1:]
5209             error_msg_s = "\n".join(error_msg) + "\n"
5210             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5211                               error_line)
5212             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5213             units = [simplejson.loads(line) for line in lines[:first_error]]
5214             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5215             last_unit = units[-1]
5216             self.failUnlessEqual(last_unit["path"], ["subdir"])
5217             r = last_unit["check-results"]["results"]
5218             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5219             self.failUnlessReallyEqual(r["count-happiness"], 1)
5220             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5221             self.failUnlessReallyEqual(r["recoverable"], False)
5222         d.addCallback(_check_broken_deepcheck)
5223
5224         d.addErrback(self.explain_web_error)
5225         return d
5226
5227     def test_deep_check_and_repair(self):
5228         self.basedir = "web/Grid/deep_check_and_repair"
5229         self.set_up_grid()
5230         c0 = self.g.clients[0]
5231         self.uris = {}
5232         self.fileurls = {}
5233         DATA = "data" * 100
5234         d = c0.create_dirnode()
5235         def _stash_root_and_create_file(n):
5236             self.rootnode = n
5237             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5238             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5239         d.addCallback(_stash_root_and_create_file)
5240         def _stash_uri(fn, which):
5241             self.uris[which] = fn.get_uri()
5242         d.addCallback(_stash_uri, "good")
5243         d.addCallback(lambda ign:
5244                       self.rootnode.add_file(u"small",
5245                                              upload.Data("literal",
5246                                                         convergence="")))
5247         d.addCallback(_stash_uri, "small")
5248         d.addCallback(lambda ign:
5249                       self.rootnode.add_file(u"sick",
5250                                              upload.Data(DATA+"1",
5251                                                         convergence="")))
5252         d.addCallback(_stash_uri, "sick")
5253         #d.addCallback(lambda ign:
5254         #              self.rootnode.add_file(u"dead",
5255         #                                     upload.Data(DATA+"2",
5256         #                                                convergence="")))
5257         #d.addCallback(_stash_uri, "dead")
5258
5259         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5260         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5261         #d.addCallback(_stash_uri, "corrupt")
5262
5263         def _clobber_shares(ignored):
5264             good_shares = self.find_uri_shares(self.uris["good"])
5265             self.failUnlessReallyEqual(len(good_shares), 10)
5266             sick_shares = self.find_uri_shares(self.uris["sick"])
5267             os.unlink(sick_shares[0][2])
5268             #dead_shares = self.find_uri_shares(self.uris["dead"])
5269             #for i in range(1, 10):
5270             #    os.unlink(dead_shares[i][2])
5271
5272             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5273             #cso = CorruptShareOptions()
5274             #cso.stdout = StringIO()
5275             #cso.parseOptions([c_shares[0][2]])
5276             #corrupt_share(cso)
5277         d.addCallback(_clobber_shares)
5278
5279         # root
5280         # root/good   CHK, 10 shares
5281         # root/small  LIT
5282         # root/sick   CHK, 9 shares
5283
5284         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5285         def _done(res):
5286             units = [simplejson.loads(line)
5287                      for line in res.splitlines()
5288                      if line]
5289             self.failUnlessReallyEqual(len(units), 4+1)
5290             # should be parent-first
5291             u0 = units[0]
5292             self.failUnlessEqual(u0["path"], [])
5293             self.failUnlessEqual(u0["type"], "directory")
5294             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5295             u0crr = u0["check-and-repair-results"]
5296             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5297             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5298             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5299
5300             ugood = [u for u in units
5301                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5302             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5303             ugoodcrr = ugood["check-and-repair-results"]
5304             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5305             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5306             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5307
5308             usick = [u for u in units
5309                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5310             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5311             usickcrr = usick["check-and-repair-results"]
5312             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5313             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5314             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5315             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5316             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5317             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5318
5319             stats = units[-1]
5320             self.failUnlessEqual(stats["type"], "stats")
5321             s = stats["stats"]
5322             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5323             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5324             self.failUnlessReallyEqual(s["count-directories"], 1)
5325         d.addCallback(_done)
5326
5327         d.addErrback(self.explain_web_error)
5328         return d
5329
5330     def _count_leases(self, ignored, which):
5331         u = self.uris[which]
5332         shares = self.find_uri_shares(u)
5333         lease_counts = []
5334         for shnum, serverid, fn in shares:
5335             sf = get_share_file(fn)
5336             num_leases = len(list(sf.get_leases()))
5337             lease_counts.append( (fn, num_leases) )
5338         return lease_counts
5339
5340     def _assert_leasecount(self, lease_counts, expected):
5341         for (fn, num_leases) in lease_counts:
5342             if num_leases != expected:
5343                 self.fail("expected %d leases, have %d, on %s" %
5344                           (expected, num_leases, fn))
5345
5346     def test_add_lease(self):
5347         self.basedir = "web/Grid/add_lease"
5348         self.set_up_grid(num_clients=2)
5349         c0 = self.g.clients[0]
5350         self.uris = {}
5351         DATA = "data" * 100
5352         d = c0.upload(upload.Data(DATA, convergence=""))
5353         def _stash_uri(ur, which):
5354             self.uris[which] = ur.get_uri()
5355         d.addCallback(_stash_uri, "one")
5356         d.addCallback(lambda ign:
5357                       c0.upload(upload.Data(DATA+"1", convergence="")))
5358         d.addCallback(_stash_uri, "two")
5359         def _stash_mutable_uri(n, which):
5360             self.uris[which] = n.get_uri()
5361             assert isinstance(self.uris[which], str)
5362         d.addCallback(lambda ign:
5363             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5364         d.addCallback(_stash_mutable_uri, "mutable")
5365
5366         def _compute_fileurls(ignored):
5367             self.fileurls = {}
5368             for which in self.uris:
5369                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5370         d.addCallback(_compute_fileurls)
5371
5372         d.addCallback(self._count_leases, "one")
5373         d.addCallback(self._assert_leasecount, 1)
5374         d.addCallback(self._count_leases, "two")
5375         d.addCallback(self._assert_leasecount, 1)
5376         d.addCallback(self._count_leases, "mutable")
5377         d.addCallback(self._assert_leasecount, 1)
5378
5379         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5380         def _got_html_good(res):
5381             self.failUnlessIn("Healthy", res)
5382             self.failIfIn("Not Healthy", res)
5383         d.addCallback(_got_html_good)
5384
5385         d.addCallback(self._count_leases, "one")
5386         d.addCallback(self._assert_leasecount, 1)
5387         d.addCallback(self._count_leases, "two")
5388         d.addCallback(self._assert_leasecount, 1)
5389         d.addCallback(self._count_leases, "mutable")
5390         d.addCallback(self._assert_leasecount, 1)
5391
5392         # this CHECK uses the original client, which uses the same
5393         # lease-secrets, so it will just renew the original lease
5394         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5395         d.addCallback(_got_html_good)
5396
5397         d.addCallback(self._count_leases, "one")
5398         d.addCallback(self._assert_leasecount, 1)
5399         d.addCallback(self._count_leases, "two")
5400         d.addCallback(self._assert_leasecount, 1)
5401         d.addCallback(self._count_leases, "mutable")
5402         d.addCallback(self._assert_leasecount, 1)
5403
5404         # this CHECK uses an alternate client, which adds a second lease
5405         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5406         d.addCallback(_got_html_good)
5407
5408         d.addCallback(self._count_leases, "one")
5409         d.addCallback(self._assert_leasecount, 2)
5410         d.addCallback(self._count_leases, "two")
5411         d.addCallback(self._assert_leasecount, 1)
5412         d.addCallback(self._count_leases, "mutable")
5413         d.addCallback(self._assert_leasecount, 1)
5414
5415         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5416         d.addCallback(_got_html_good)
5417
5418         d.addCallback(self._count_leases, "one")
5419         d.addCallback(self._assert_leasecount, 2)
5420         d.addCallback(self._count_leases, "two")
5421         d.addCallback(self._assert_leasecount, 1)
5422         d.addCallback(self._count_leases, "mutable")
5423         d.addCallback(self._assert_leasecount, 1)
5424
5425         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5426                       clientnum=1)
5427         d.addCallback(_got_html_good)
5428
5429         d.addCallback(self._count_leases, "one")
5430         d.addCallback(self._assert_leasecount, 2)
5431         d.addCallback(self._count_leases, "two")
5432         d.addCallback(self._assert_leasecount, 1)
5433         d.addCallback(self._count_leases, "mutable")
5434         d.addCallback(self._assert_leasecount, 2)
5435
5436         d.addErrback(self.explain_web_error)
5437         return d
5438
5439     def test_deep_add_lease(self):
5440         self.basedir = "web/Grid/deep_add_lease"
5441         self.set_up_grid(num_clients=2)
5442         c0 = self.g.clients[0]
5443         self.uris = {}
5444         self.fileurls = {}
5445         DATA = "data" * 100
5446         d = c0.create_dirnode()
5447         def _stash_root_and_create_file(n):
5448             self.rootnode = n
5449             self.uris["root"] = n.get_uri()
5450             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5451             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5452         d.addCallback(_stash_root_and_create_file)
5453         def _stash_uri(fn, which):
5454             self.uris[which] = fn.get_uri()
5455         d.addCallback(_stash_uri, "one")
5456         d.addCallback(lambda ign:
5457                       self.rootnode.add_file(u"small",
5458                                              upload.Data("literal",
5459                                                         convergence="")))
5460         d.addCallback(_stash_uri, "small")
5461
5462         d.addCallback(lambda ign:
5463             c0.create_mutable_file(publish.MutableData("mutable")))
5464         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5465         d.addCallback(_stash_uri, "mutable")
5466
5467         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5468         def _done(res):
5469             units = [simplejson.loads(line)
5470                      for line in res.splitlines()
5471                      if line]
5472             # root, one, small, mutable,   stats
5473             self.failUnlessReallyEqual(len(units), 4+1)
5474         d.addCallback(_done)
5475
5476         d.addCallback(self._count_leases, "root")
5477         d.addCallback(self._assert_leasecount, 1)
5478         d.addCallback(self._count_leases, "one")
5479         d.addCallback(self._assert_leasecount, 1)
5480         d.addCallback(self._count_leases, "mutable")
5481         d.addCallback(self._assert_leasecount, 1)
5482
5483         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5484         d.addCallback(_done)
5485
5486         d.addCallback(self._count_leases, "root")
5487         d.addCallback(self._assert_leasecount, 1)
5488         d.addCallback(self._count_leases, "one")
5489         d.addCallback(self._assert_leasecount, 1)
5490         d.addCallback(self._count_leases, "mutable")
5491         d.addCallback(self._assert_leasecount, 1)
5492
5493         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5494                       clientnum=1)
5495         d.addCallback(_done)
5496
5497         d.addCallback(self._count_leases, "root")
5498         d.addCallback(self._assert_leasecount, 2)
5499         d.addCallback(self._count_leases, "one")
5500         d.addCallback(self._assert_leasecount, 2)
5501         d.addCallback(self._count_leases, "mutable")
5502         d.addCallback(self._assert_leasecount, 2)
5503
5504         d.addErrback(self.explain_web_error)
5505         return d
5506
5507
5508     def test_exceptions(self):
5509         self.basedir = "web/Grid/exceptions"
5510         self.set_up_grid(num_clients=1, num_servers=2)
5511         c0 = self.g.clients[0]
5512         c0.encoding_params['happy'] = 2
5513         self.fileurls = {}
5514         DATA = "data" * 100
5515         d = c0.create_dirnode()
5516         def _stash_root(n):
5517             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5518             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5519             return n
5520         d.addCallback(_stash_root)
5521         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5522         def _stash_bad(ur):
5523             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5524             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5525
5526             u = uri.from_string(ur.get_uri())
5527             u.key = testutil.flip_bit(u.key, 0)
5528             baduri = u.to_string()
5529             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5530         d.addCallback(_stash_bad)
5531         d.addCallback(lambda ign: c0.create_dirnode())
5532         def _mangle_dirnode_1share(n):
5533             u = n.get_uri()
5534             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5535             self.fileurls["dir-1share-json"] = url + "?t=json"
5536             self.delete_shares_numbered(u, range(1,10))
5537         d.addCallback(_mangle_dirnode_1share)
5538         d.addCallback(lambda ign: c0.create_dirnode())
5539         def _mangle_dirnode_0share(n):
5540             u = n.get_uri()
5541             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5542             self.fileurls["dir-0share-json"] = url + "?t=json"
5543             self.delete_shares_numbered(u, range(0,10))
5544         d.addCallback(_mangle_dirnode_0share)
5545
5546         # NotEnoughSharesError should be reported sensibly, with a
5547         # text/plain explanation of the problem, and perhaps some
5548         # information on which shares *could* be found.
5549
5550         d.addCallback(lambda ignored:
5551                       self.shouldHTTPError("GET unrecoverable",
5552                                            410, "Gone", "NoSharesError",
5553                                            self.GET, self.fileurls["0shares"]))
5554         def _check_zero_shares(body):
5555             self.failIfIn("<html>", body)
5556             body = " ".join(body.strip().split())
5557             exp = ("NoSharesError: no shares could be found. "
5558                    "Zero shares usually indicates a corrupt URI, or that "
5559                    "no servers were connected, but it might also indicate "
5560                    "severe corruption. You should perform a filecheck on "
5561                    "this object to learn more. The full error message is: "
5562                    "no shares (need 3). Last failure: None")
5563             self.failUnlessReallyEqual(exp, body)
5564         d.addCallback(_check_zero_shares)
5565
5566
5567         d.addCallback(lambda ignored:
5568                       self.shouldHTTPError("GET 1share",
5569                                            410, "Gone", "NotEnoughSharesError",
5570                                            self.GET, self.fileurls["1share"]))
5571         def _check_one_share(body):
5572             self.failIfIn("<html>", body)
5573             body = " ".join(body.strip().split())
5574             msgbase = ("NotEnoughSharesError: This indicates that some "
5575                        "servers were unavailable, or that shares have been "
5576                        "lost to server departure, hard drive failure, or disk "
5577                        "corruption. You should perform a filecheck on "
5578                        "this object to learn more. The full error message is:"
5579                        )
5580             msg1 = msgbase + (" ran out of shares:"
5581                               " complete=sh0"
5582                               " pending="
5583                               " overdue= unused= need 3. Last failure: None")
5584             msg2 = msgbase + (" ran out of shares:"
5585                               " complete="
5586                               " pending=Share(sh0-on-xgru5)"
5587                               " overdue= unused= need 3. Last failure: None")
5588             self.failUnless(body == msg1 or body == msg2, body)
5589         d.addCallback(_check_one_share)
5590
5591         d.addCallback(lambda ignored:
5592                       self.shouldHTTPError("GET imaginary",
5593                                            404, "Not Found", None,
5594                                            self.GET, self.fileurls["imaginary"]))
5595         def _missing_child(body):
5596             self.failUnlessIn("No such child: imaginary", body)
5597         d.addCallback(_missing_child)
5598
5599         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5600         def _check_0shares_dir_html(body):
5601             self.failUnlessIn(DIR_HTML_TAG, body)
5602             # we should see the regular page, but without the child table or
5603             # the dirops forms
5604             body = " ".join(body.strip().split())
5605             self.failUnlessIn('href="?t=info">More info on this directory',
5606                               body)
5607             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5608                    "could not be retrieved, because there were insufficient "
5609                    "good shares. This might indicate that no servers were "
5610                    "connected, insufficient servers were connected, the URI "
5611                    "was corrupt, or that shares have been lost due to server "
5612                    "departure, hard drive failure, or disk corruption. You "
5613                    "should perform a filecheck on this object to learn more.")
5614             self.failUnlessIn(exp, body)
5615             self.failUnlessIn("No upload forms: directory is unreadable", body)
5616         d.addCallback(_check_0shares_dir_html)
5617
5618         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5619         def _check_1shares_dir_html(body):
5620             # at some point, we'll split UnrecoverableFileError into 0-shares
5621             # and some-shares like we did for immutable files (since there
5622             # are different sorts of advice to offer in each case). For now,
5623             # they present the same way.
5624             self.failUnlessIn(DIR_HTML_TAG, body)
5625             body = " ".join(body.strip().split())
5626             self.failUnlessIn('href="?t=info">More info on this directory',
5627                               body)
5628             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5629                    "could not be retrieved, because there were insufficient "
5630                    "good shares. This might indicate that no servers were "
5631                    "connected, insufficient servers were connected, the URI "
5632                    "was corrupt, or that shares have been lost due to server "
5633                    "departure, hard drive failure, or disk corruption. You "
5634                    "should perform a filecheck on this object to learn more.")
5635             self.failUnlessIn(exp, body)
5636             self.failUnlessIn("No upload forms: directory is unreadable", body)
5637         d.addCallback(_check_1shares_dir_html)
5638
5639         d.addCallback(lambda ignored:
5640                       self.shouldHTTPError("GET dir-0share-json",
5641                                            410, "Gone", "UnrecoverableFileError",
5642                                            self.GET,
5643                                            self.fileurls["dir-0share-json"]))
5644         def _check_unrecoverable_file(body):
5645             self.failIfIn("<html>", body)
5646             body = " ".join(body.strip().split())
5647             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5648                    "could not be retrieved, because there were insufficient "
5649                    "good shares. This might indicate that no servers were "
5650                    "connected, insufficient servers were connected, the URI "
5651                    "was corrupt, or that shares have been lost due to server "
5652                    "departure, hard drive failure, or disk corruption. You "
5653                    "should perform a filecheck on this object to learn more.")
5654             self.failUnlessReallyEqual(exp, body)
5655         d.addCallback(_check_unrecoverable_file)
5656
5657         d.addCallback(lambda ignored:
5658                       self.shouldHTTPError("GET dir-1share-json",
5659                                            410, "Gone", "UnrecoverableFileError",
5660                                            self.GET,
5661                                            self.fileurls["dir-1share-json"]))
5662         d.addCallback(_check_unrecoverable_file)
5663
5664         d.addCallback(lambda ignored:
5665                       self.shouldHTTPError("GET imaginary",
5666                                            404, "Not Found", None,
5667                                            self.GET, self.fileurls["imaginary"]))
5668
5669         # attach a webapi child that throws a random error, to test how it
5670         # gets rendered.
5671         w = c0.getServiceNamed("webish")
5672         w.root.putChild("ERRORBOOM", ErrorBoom())
5673
5674         # "Accept: */*" :        should get a text/html stack trace
5675         # "Accept: text/plain" : should get a text/plain stack trace
5676         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5677         # no Accept header:      should get a text/html stack trace
5678
5679         d.addCallback(lambda ignored:
5680                       self.shouldHTTPError("GET errorboom_html",
5681                                            500, "Internal Server Error", None,
5682                                            self.GET, "ERRORBOOM",
5683                                            headers={"accept": "*/*"}))
5684         def _internal_error_html1(body):
5685             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5686         d.addCallback(_internal_error_html1)
5687
5688         d.addCallback(lambda ignored:
5689                       self.shouldHTTPError("GET errorboom_text",
5690                                            500, "Internal Server Error", None,
5691                                            self.GET, "ERRORBOOM",
5692                                            headers={"accept": "text/plain"}))
5693         def _internal_error_text2(body):
5694             self.failIfIn("<html>", body)
5695             self.failUnless(body.startswith("Traceback "), body)
5696         d.addCallback(_internal_error_text2)
5697
5698         CLI_accepts = "text/plain, application/octet-stream"
5699         d.addCallback(lambda ignored:
5700                       self.shouldHTTPError("GET errorboom_text",
5701                                            500, "Internal Server Error", None,
5702                                            self.GET, "ERRORBOOM",
5703                                            headers={"accept": CLI_accepts}))
5704         def _internal_error_text3(body):
5705             self.failIfIn("<html>", body)
5706             self.failUnless(body.startswith("Traceback "), body)
5707         d.addCallback(_internal_error_text3)
5708
5709         d.addCallback(lambda ignored:
5710                       self.shouldHTTPError("GET errorboom_text",
5711                                            500, "Internal Server Error", None,
5712                                            self.GET, "ERRORBOOM"))
5713         def _internal_error_html4(body):
5714             self.failUnlessIn("<html>", body)
5715         d.addCallback(_internal_error_html4)
5716
5717         def _flush_errors(res):
5718             # Trial: please ignore the CompletelyUnhandledError in the logs
5719             self.flushLoggedErrors(CompletelyUnhandledError)
5720             return res
5721         d.addBoth(_flush_errors)
5722
5723         return d
5724
5725     def test_blacklist(self):
5726         # download from a blacklisted URI, get an error
5727         self.basedir = "web/Grid/blacklist"
5728         self.set_up_grid()
5729         c0 = self.g.clients[0]
5730         c0_basedir = c0.basedir
5731         fn = os.path.join(c0_basedir, "access.blacklist")
5732         self.uris = {}
5733         DATA = "off-limits " * 50
5734
5735         d = c0.upload(upload.Data(DATA, convergence=""))
5736         def _stash_uri_and_create_dir(ur):
5737             self.uri = ur.get_uri()
5738             self.url = "uri/"+self.uri
5739             u = uri.from_string_filenode(self.uri)
5740             self.si = u.get_storage_index()
5741             childnode = c0.create_node_from_uri(self.uri, None)
5742             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5743         d.addCallback(_stash_uri_and_create_dir)
5744         def _stash_dir(node):
5745             self.dir_node = node
5746             self.dir_uri = node.get_uri()
5747             self.dir_url = "uri/"+self.dir_uri
5748         d.addCallback(_stash_dir)
5749         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5750         def _check_dir_html(body):
5751             self.failUnlessIn(DIR_HTML_TAG, body)
5752             self.failUnlessIn("blacklisted.txt</a>", body)
5753         d.addCallback(_check_dir_html)
5754         d.addCallback(lambda ign: self.GET(self.url))
5755         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5756
5757         def _blacklist(ign):
5758             f = open(fn, "w")
5759             f.write(" # this is a comment\n")
5760             f.write(" \n")
5761             f.write("\n") # also exercise blank lines
5762             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5763             f.close()
5764             # clients should be checking the blacklist each time, so we don't
5765             # need to restart the client
5766         d.addCallback(_blacklist)
5767         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5768                                                        403, "Forbidden",
5769                                                        "Access Prohibited: off-limits",
5770                                                        self.GET, self.url))
5771
5772         # We should still be able to list the parent directory, in HTML...
5773         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5774         def _check_dir_html2(body):
5775             self.failUnlessIn(DIR_HTML_TAG, body)
5776             self.failUnlessIn("blacklisted.txt</strike>", body)
5777         d.addCallback(_check_dir_html2)
5778
5779         # ... and in JSON (used by CLI).
5780         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5781         def _check_dir_json(res):
5782             data = simplejson.loads(res)
5783             self.failUnless(isinstance(data, list), data)
5784             self.failUnlessEqual(data[0], "dirnode")
5785             self.failUnless(isinstance(data[1], dict), data)
5786             self.failUnlessIn("children", data[1])
5787             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5788             childdata = data[1]["children"]["blacklisted.txt"]
5789             self.failUnless(isinstance(childdata, list), data)
5790             self.failUnlessEqual(childdata[0], "filenode")
5791             self.failUnless(isinstance(childdata[1], dict), data)
5792         d.addCallback(_check_dir_json)
5793
5794         def _unblacklist(ign):
5795             open(fn, "w").close()
5796             # the Blacklist object watches mtime to tell when the file has
5797             # changed, but on windows this test will run faster than the
5798             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5799             # to force a reload.
5800             self.g.clients[0].blacklist.last_mtime -= 2.0
5801         d.addCallback(_unblacklist)
5802
5803         # now a read should work
5804         d.addCallback(lambda ign: self.GET(self.url))
5805         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5806
5807         # read again to exercise the blacklist-is-unchanged logic
5808         d.addCallback(lambda ign: self.GET(self.url))
5809         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5810
5811         # now add a blacklisted directory, and make sure files under it are
5812         # refused too
5813         def _add_dir(ign):
5814             childnode = c0.create_node_from_uri(self.uri, None)
5815             return c0.create_dirnode({u"child": (childnode,{}) })
5816         d.addCallback(_add_dir)
5817         def _get_dircap(dn):
5818             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5819             self.dir_url_base = "uri/"+dn.get_write_uri()
5820             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5821             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5822             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5823             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5824         d.addCallback(_get_dircap)
5825         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5826         d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5827         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5828         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5829         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5830         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5831         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5832         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5833         d.addCallback(lambda ign: self.GET(self.child_url))
5834         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5835
5836         def _block_dir(ign):
5837             f = open(fn, "w")
5838             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5839             f.close()
5840             self.g.clients[0].blacklist.last_mtime -= 2.0
5841         d.addCallback(_block_dir)
5842         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5843                                                        403, "Forbidden",
5844                                                        "Access Prohibited: dir-off-limits",
5845                                                        self.GET, self.dir_url_base))
5846         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5847                                                        403, "Forbidden",
5848                                                        "Access Prohibited: dir-off-limits",
5849                                                        self.GET, self.dir_url_json1))
5850         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5851                                                        403, "Forbidden",
5852                                                        "Access Prohibited: dir-off-limits",
5853                                                        self.GET, self.dir_url_json2))
5854         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5855                                                        403, "Forbidden",
5856                                                        "Access Prohibited: dir-off-limits",
5857                                                        self.GET, self.dir_url_json_ro))
5858         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5859                                                        403, "Forbidden",
5860                                                        "Access Prohibited: dir-off-limits",
5861                                                        self.GET, self.child_url))
5862         return d
5863
5864
5865 class CompletelyUnhandledError(Exception):
5866     pass
5867 class ErrorBoom(rend.Page):
5868     def beforeRender(self, ctx):
5869         raise CompletelyUnhandledError("whoops")