src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649         d.addCallback(_check_dl_json)
 650         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 651         def _check_ul(res):
 652             self.failUnless("File Upload Status" in res, res)
 653         d.addCallback(_check_ul)
 654         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 655         def _check_mapupdate(res):
 656             self.failUnless("Mutable File Servermap Update Status" in res, res)
 657         d.addCallback(_check_mapupdate)
 658         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 659         def _check_publish(res):
 660             self.failUnless("Mutable File Publish Status" in res, res)
 661         d.addCallback(_check_publish)
 662         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 663         def _check_retrieve(res):
 664             self.failUnless("Mutable File Retrieve Status" in res, res)
 665         d.addCallback(_check_retrieve)
 666
 667         return d
 668
 669     def test_status_numbers(self):
 670         drrm = status.DownloadResultsRendererMixin()
 671         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 680
 681         urrm = status.UploadResultsRendererMixin()
 682         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 691
 692     def test_GET_FILEURL(self):
 693         d = self.GET(self.public_url + "/foo/bar.txt")
 694         d.addCallback(self.failUnlessIsBarDotTxt)
 695         return d
 696
 697     def test_GET_FILEURL_range(self):
 698         headers = {"range": "bytes=1-10"}
 699         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 700                      return_response=True)
 701         def _got((res, status, headers)):
 702             self.failUnlessReallyEqual(int(status), 206)
 703             self.failUnless(headers.has_key("content-range"))
 704             self.failUnlessReallyEqual(headers["content-range"][0],
 705                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 706             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 707         d.addCallback(_got)
 708         return d
 709
 710     def test_GET_FILEURL_partial_range(self):
 711         headers = {"range": "bytes=5-"}
 712         length  = len(self.BAR_CONTENTS)
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 206)
 717             self.failUnless(headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(headers["content-range"][0],
 719                                        "bytes 5-%d/%d" % (length-1, length))
 720             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 721         d.addCallback(_got)
 722         return d
 723
 724     def test_GET_FILEURL_partial_end_range(self):
 725         headers = {"range": "bytes=-5"}
 726         length  = len(self.BAR_CONTENTS)
 727         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 728                      return_response=True)
 729         def _got((res, status, headers)):
 730             self.failUnlessReallyEqual(int(status), 206)
 731             self.failUnless(headers.has_key("content-range"))
 732             self.failUnlessReallyEqual(headers["content-range"][0],
 733                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 734             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_partial_range_overrun(self):
 739         headers = {"range": "bytes=100-200"}
 740         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 741                              "416 Requested Range not satisfiable",
 742                              "First beyond end of file",
 743                              self.GET, self.public_url + "/foo/bar.txt",
 744                              headers=headers)
 745         return d
 746
 747     def test_HEAD_FILEURL_range(self):
 748         headers = {"range": "bytes=1-10"}
 749         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 750                      return_response=True)
 751         def _got((res, status, headers)):
 752             self.failUnlessReallyEqual(res, "")
 753             self.failUnlessReallyEqual(int(status), 206)
 754             self.failUnless(headers.has_key("content-range"))
 755             self.failUnlessReallyEqual(headers["content-range"][0],
 756                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_HEAD_FILEURL_partial_range(self):
 761         headers = {"range": "bytes=5-"}
 762         length  = len(self.BAR_CONTENTS)
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 764                      return_response=True)
 765         def _got((res, status, headers)):
 766             self.failUnlessReallyEqual(int(status), 206)
 767             self.failUnless(headers.has_key("content-range"))
 768             self.failUnlessReallyEqual(headers["content-range"][0],
 769                                        "bytes 5-%d/%d" % (length-1, length))
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_HEAD_FILEURL_partial_end_range(self):
 774         headers = {"range": "bytes=-5"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range_overrun(self):
 787         headers = {"range": "bytes=100-200"}
 788         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 789                              "416 Requested Range not satisfiable",
 790                              "",
 791                              self.HEAD, self.public_url + "/foo/bar.txt",
 792                              headers=headers)
 793         return d
 794
 795     def test_GET_FILEURL_range_bad(self):
 796         headers = {"range": "BOGUS=fizbop-quarnak"}
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 200)
 801             self.failUnless(not headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 803         d.addCallback(_got)
 804         return d
 805
 806     def test_HEAD_FILEURL(self):
 807         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 808         def _got((res, status, headers)):
 809             self.failUnlessReallyEqual(res, "")
 810             self.failUnlessReallyEqual(headers["content-length"][0],
 811                                        str(len(self.BAR_CONTENTS)))
 812             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 813         d.addCallback(_got)
 814         return d
 815
 816     def test_GET_FILEURL_named(self):
 817         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 818         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 819         d = self.GET(base + "/@@name=/blah.txt")
 820         d.addCallback(self.failUnlessIsBarDotTxt)
 821         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 822         d.addCallback(self.failUnlessIsBarDotTxt)
 823         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 824         d.addCallback(self.failUnlessIsBarDotTxt)
 825         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 826         d.addCallback(self.failUnlessIsBarDotTxt)
 827         save_url = base + "?save=true&filename=blah.txt"
 828         d.addCallback(lambda res: self.GET(save_url))
 829         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 830         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 831         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 832         u_url = base + "?save=true&filename=" + u_fn_e
 833         d.addCallback(lambda res: self.GET(u_url))
 834         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 835         return d
 836
 837     def test_PUT_FILEURL_named_bad(self):
 838         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 839         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 840                              "400 Bad Request",
 841                              "/file can only be used with GET or HEAD",
 842                              self.PUT, base + "/@@name=/blah.txt", "")
 843         return d
 844
 845
 846     def test_GET_DIRURL_named_bad(self):
 847         base = "/file/%s" % urllib.quote(self._foo_uri)
 848         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 849                              "400 Bad Request",
 850                              "is not a file-cap",
 851                              self.GET, base + "/@@name=/blah.txt")
 852         return d
 853
 854     def test_GET_slash_file_bad(self):
 855         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 856                              "404 Not Found",
 857                              "/file must be followed by a file-cap and a name",
 858                              self.GET, "/file")
 859         return d
 860
 861     def test_GET_unhandled_URI_named(self):
 862         contents, n, newuri = self.makefile(12)
 863         verifier_cap = n.get_verify_cap().to_string()
 864         base = "/file/%s" % urllib.quote(verifier_cap)
 865         # client.create_node_from_uri() can't handle verify-caps
 866         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 867                              "400 Bad Request", "is not a file-cap",
 868                              self.GET, base)
 869         return d
 870
 871     def test_GET_unhandled_URI(self):
 872         contents, n, newuri = self.makefile(12)
 873         verifier_cap = n.get_verify_cap().to_string()
 874         base = "/uri/%s" % urllib.quote(verifier_cap)
 875         # client.create_node_from_uri() can't handle verify-caps
 876         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 877                              "400 Bad Request",
 878                              "GET unknown URI type: can only do t=info",
 879                              self.GET, base)
 880         return d
 881
 882     def test_GET_FILE_URI(self):
 883         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 884         d = self.GET(base)
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         return d
 887
 888     def test_GET_FILE_URI_mdmf(self):
 889         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 890         d = self.GET(base)
 891         d.addCallback(self.failUnlessIsQuuxDotTxt)
 892         return d
 893
 894     def test_GET_FILE_URI_mdmf_extensions(self):
 895         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 896         d = self.GET(base)
 897         d.addCallback(self.failUnlessIsQuuxDotTxt)
 898         return d
 899
 900     def test_GET_FILE_URI_mdmf_readonly(self):
 901         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 902         d = self.GET(base)
 903         d.addCallback(self.failUnlessIsQuuxDotTxt)
 904         return d
 905
 906     def test_GET_FILE_URI_badchild(self):
 907         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 908         errmsg = "Files have no children, certainly not named 'boguschild'"
 909         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 910                              "400 Bad Request", errmsg,
 911                              self.GET, base)
 912         return d
 913
 914     def test_PUT_FILE_URI_badchild(self):
 915         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 916         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 917         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 918                              "400 Bad Request", errmsg,
 919                              self.PUT, base, "")
 920         return d
 921
 922     def test_PUT_FILE_URI_mdmf(self):
 923         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 924         self._quux_new_contents = "new_contents"
 925         d = self.GET(base)
 926         d.addCallback(lambda res:
 927             self.failUnlessIsQuuxDotTxt(res))
 928         d.addCallback(lambda ignored:
 929             self.PUT(base, self._quux_new_contents))
 930         d.addCallback(lambda ignored:
 931             self.GET(base))
 932         d.addCallback(lambda res:
 933             self.failUnlessReallyEqual(res, self._quux_new_contents))
 934         return d
 935
 936     def test_PUT_FILE_URI_mdmf_extensions(self):
 937         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 938         self._quux_new_contents = "new_contents"
 939         d = self.GET(base)
 940         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 941         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 942         d.addCallback(lambda ignored: self.GET(base))
 943         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 944                                                        res))
 945         return d
 946
 947     def test_PUT_FILE_URI_mdmf_readonly(self):
 948         # We're not allowed to PUT things to a readonly cap.
 949         base = "/uri/%s" % self._quux_txt_readonly_uri
 950         d = self.GET(base)
 951         d.addCallback(lambda res:
 952             self.failUnlessIsQuuxDotTxt(res))
 953         # What should we get here? We get a 500 error now; that's not right.
 954         d.addCallback(lambda ignored:
 955             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 956                              "400 Bad Request", "read-only cap",
 957                              self.PUT, base, "new data"))
 958         return d
 959
 960     def test_PUT_FILE_URI_sdmf_readonly(self):
 961         # We're not allowed to put things to a readonly cap.
 962         base = "/uri/%s" % self._baz_txt_readonly_uri
 963         d = self.GET(base)
 964         d.addCallback(lambda res:
 965             self.failUnlessIsBazDotTxt(res))
 966         d.addCallback(lambda ignored:
 967             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 968                              "400 Bad Request", "read-only cap",
 969                              self.PUT, base, "new_data"))
 970         return d
 971
 972     # TODO: version of this with a Unicode filename
 973     def test_GET_FILEURL_save(self):
 974         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 975                      return_response=True)
 976         def _got((res, statuscode, headers)):
 977             content_disposition = headers["content-disposition"][0]
 978             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 979             self.failUnlessIsBarDotTxt(res)
 980         d.addCallback(_got)
 981         return d
 982
 983     def test_GET_FILEURL_missing(self):
 984         d = self.GET(self.public_url + "/foo/missing")
 985         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 986         return d
 987
 988     def test_GET_FILEURL_info_mdmf(self):
 989         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 990         def _got(res):
 991             self.failUnlessIn("mutable file (mdmf)", res)
 992             self.failUnlessIn(self._quux_txt_uri, res)
 993             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 994         d.addCallback(_got)
 995         return d
 996
 997     def test_GET_FILEURL_info_mdmf_readonly(self):
 998         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 999         def _got(res):
1000             self.failUnlessIn("mutable file (mdmf)", res)
1001             self.failIfIn(self._quux_txt_uri, res)
1002             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1003         d.addCallback(_got)
1004         return d
1005
1006     def test_GET_FILEURL_info_sdmf(self):
1007         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1008         def _got(res):
1009             self.failUnlessIn("mutable file (sdmf)", res)
1010             self.failUnlessIn(self._baz_txt_uri, res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_info_mdmf_extensions(self):
1015         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1016         def _got(res):
1017             self.failUnlessIn("mutable file (mdmf)", res)
1018             self.failUnlessIn(self._quux_txt_uri, res)
1019             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_PUT_overwrite_only_files(self):
1024         # create a directory, put a file in that directory.
1025         contents, n, filecap = self.makefile(8)
1026         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1027         d.addCallback(lambda res:
1028             self.PUT(self.public_url + "/foo/dir/file1.txt",
1029                      self.NEWFILE_CONTENTS))
1030         # try to overwrite the file with replace=only-files
1031         # (this should work)
1032         d.addCallback(lambda res:
1033             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1034                      filecap))
1035         d.addCallback(lambda res:
1036             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1037                  "There was already a child by that name, and you asked me "
1038                  "to not replace it",
1039                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1040                  filecap))
1041         return d
1042
1043     def test_PUT_NEWFILEURL(self):
1044         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1045         # TODO: we lose the response code, so we can't check this
1046         #self.failUnlessReallyEqual(responsecode, 201)
1047         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1048         d.addCallback(lambda res:
1049                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1050                                                       self.NEWFILE_CONTENTS))
1051         return d
1052
1053     def test_PUT_NEWFILEURL_not_mutable(self):
1054         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1055                      self.NEWFILE_CONTENTS)
1056         # TODO: we lose the response code, so we can't check this
1057         #self.failUnlessReallyEqual(responsecode, 201)
1058         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1059         d.addCallback(lambda res:
1060                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1061                                                       self.NEWFILE_CONTENTS))
1062         return d
1063
1064     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1065         # this should get us a few segments of an MDMF mutable file,
1066         # which we can then test for.
1067         contents = self.NEWFILE_CONTENTS * 300000
1068         d = self.PUT("/uri?format=mdmf",
1069                      contents)
1070         def _got_filecap(filecap):
1071             self.failUnless(filecap.startswith("URI:MDMF"))
1072             return filecap
1073         d.addCallback(_got_filecap)
1074         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1075         d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1076         return d
1077
1078     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1079         contents = self.NEWFILE_CONTENTS * 300000
1080         d = self.PUT("/uri?format=sdmf",
1081                      contents)
1082         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1083         d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1084         return d
1085
1086     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1087         contents = self.NEWFILE_CONTENTS * 300000
1088         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1089                                     400, "Bad Request", "Unknown format: foo",
1090                                     self.PUT, "/uri?format=foo",
1091                                     contents)
1092
1093     def test_PUT_NEWFILEURL_range_bad(self):
1094         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1095         target = self.public_url + "/foo/new.txt"
1096         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1097                              "501 Not Implemented",
1098                              "Content-Range in PUT not yet supported",
1099                              # (and certainly not for immutable files)
1100                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1101                              headers=headers)
1102         d.addCallback(lambda res:
1103                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1104         return d
1105
1106     def test_PUT_NEWFILEURL_mutable(self):
1107         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1108                      self.NEWFILE_CONTENTS)
1109         # TODO: we lose the response code, so we can't check this
1110         #self.failUnlessReallyEqual(responsecode, 201)
1111         def _check_uri(res):
1112             u = uri.from_string_mutable_filenode(res)
1113             self.failUnless(u.is_mutable())
1114             self.failIf(u.is_readonly())
1115             return res
1116         d.addCallback(_check_uri)
1117         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1118         d.addCallback(lambda res:
1119                       self.failUnlessMutableChildContentsAre(self._foo_node,
1120                                                              u"new.txt",
1121                                                              self.NEWFILE_CONTENTS))
1122         return d
1123
1124     def test_PUT_NEWFILEURL_mutable_toobig(self):
1125         # It is okay to upload large mutable files, so we should be able
1126         # to do that.
1127         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1128                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1129         return d
1130
1131     def test_PUT_NEWFILEURL_replace(self):
1132         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1133         # TODO: we lose the response code, so we can't check this
1134         #self.failUnlessReallyEqual(responsecode, 200)
1135         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1136         d.addCallback(lambda res:
1137                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1138                                                       self.NEWFILE_CONTENTS))
1139         return d
1140
1141     def test_PUT_NEWFILEURL_bad_t(self):
1142         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1143                              "PUT to a file: bad t=bogus",
1144                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1145                              "contents")
1146         return d
1147
1148     def test_PUT_NEWFILEURL_no_replace(self):
1149         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1150                      self.NEWFILE_CONTENTS)
1151         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1152                   "409 Conflict",
1153                   "There was already a child by that name, and you asked me "
1154                   "to not replace it")
1155         return d
1156
1157     def test_PUT_NEWFILEURL_mkdirs(self):
1158         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1159         fn = self._foo_node
1160         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1161         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1162         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1163         d.addCallback(lambda res:
1164                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1165                                                       self.NEWFILE_CONTENTS))
1166         return d
1167
1168     def test_PUT_NEWFILEURL_blocked(self):
1169         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1170                      self.NEWFILE_CONTENTS)
1171         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1172                   "409 Conflict",
1173                   "Unable to create directory 'blockingfile': a file was in the way")
1174         return d
1175
1176     def test_PUT_NEWFILEURL_emptyname(self):
1177         # an empty pathname component (i.e. a double-slash) is disallowed
1178         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1179                              "400 Bad Request",
1180                              "The webapi does not allow empty pathname components",
1181                              self.PUT, self.public_url + "/foo//new.txt", "")
1182         return d
1183
1184     def test_DELETE_FILEURL(self):
1185         d = self.DELETE(self.public_url + "/foo/bar.txt")
1186         d.addCallback(lambda res:
1187                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1188         return d
1189
1190     def test_DELETE_FILEURL_missing(self):
1191         d = self.DELETE(self.public_url + "/foo/missing")
1192         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1193         return d
1194
1195     def test_DELETE_FILEURL_missing2(self):
1196         d = self.DELETE(self.public_url + "/missing/missing")
1197         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1198         return d
1199
1200     def failUnlessHasBarDotTxtMetadata(self, res):
1201         data = simplejson.loads(res)
1202         self.failUnless(isinstance(data, list))
1203         self.failUnlessIn("metadata", data[1])
1204         self.failUnlessIn("tahoe", data[1]["metadata"])
1205         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1206         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1207         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1208                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1209
1210     def test_GET_FILEURL_json(self):
1211         # twisted.web.http.parse_qs ignores any query args without an '=', so
1212         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1213         # instead. This may make it tricky to emulate the S3 interface
1214         # completely.
1215         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1216         def _check1(data):
1217             self.failUnlessIsBarJSON(data)
1218             self.failUnlessHasBarDotTxtMetadata(data)
1219             return
1220         d.addCallback(_check1)
1221         return d
1222
1223     def test_GET_FILEURL_json_mutable_type(self):
1224         # The JSON should include format, which says whether the
1225         # file is SDMF or MDMF
1226         d = self.PUT("/uri?format=mdmf",
1227                      self.NEWFILE_CONTENTS * 300000)
1228         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1229         def _got_json(json, version):
1230             data = simplejson.loads(json)
1231             assert "filenode" == data[0]
1232             data = data[1]
1233             assert isinstance(data, dict)
1234
1235             self.failUnlessIn("format", data)
1236             self.failUnlessEqual(data["format"], version)
1237
1238         d.addCallback(_got_json, "mdmf")
1239         # Now make an SDMF file and check that it is reported correctly.
1240         d.addCallback(lambda ignored:
1241             self.PUT("/uri?format=sdmf",
1242                       self.NEWFILE_CONTENTS * 300000))
1243         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244         d.addCallback(_got_json, "sdmf")
1245         return d
1246
1247     def test_GET_FILEURL_json_mdmf(self):
1248         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1249         d.addCallback(self.failUnlessIsQuuxJSON)
1250         return d
1251
1252     def test_GET_FILEURL_json_missing(self):
1253         d = self.GET(self.public_url + "/foo/missing?json")
1254         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1255         return d
1256
1257     def test_GET_FILEURL_uri(self):
1258         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1259         def _check(res):
1260             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1261         d.addCallback(_check)
1262         d.addCallback(lambda res:
1263                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1264         def _check2(res):
1265             # for now, for files, uris and readonly-uris are the same
1266             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1267         d.addCallback(_check2)
1268         return d
1269
1270     def test_GET_FILEURL_badtype(self):
1271         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1272                                  "bad t=bogus",
1273                                  self.GET,
1274                                  self.public_url + "/foo/bar.txt?t=bogus")
1275         return d
1276
1277     def test_CSS_FILE(self):
1278         d = self.GET("/tahoe_css", followRedirect=True)
1279         def _check(res):
1280             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1281             self.failUnless(CSS_STYLE.search(res), res)
1282         d.addCallback(_check)
1283         return d
1284
1285     def test_GET_FILEURL_uri_missing(self):
1286         d = self.GET(self.public_url + "/foo/missing?t=uri")
1287         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1288         return d
1289
1290     def test_GET_DIRECTORY_html(self):
1291         d = self.GET(self.public_url + "/foo", followRedirect=True)
1292         def _check(res):
1293             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1294             # These are radio buttons that allow a user to toggle
1295             # whether a particular mutable file is SDMF or MDMF.
1296             self.failUnlessIn("mutable-type-mdmf", res)
1297             self.failUnlessIn("mutable-type-sdmf", res)
1298             # Similarly, these toggle whether a particular directory
1299             # should be MDMF or SDMF.
1300             self.failUnlessIn("mutable-directory-mdmf", res)
1301             self.failUnlessIn("mutable-directory-sdmf", res)
1302             self.failUnlessIn("quux", res)
1303         d.addCallback(_check)
1304         return d
1305
1306     def test_GET_root_html(self):
1307         # make sure that we have the option to upload an unlinked
1308         # mutable file in SDMF and MDMF formats.
1309         d = self.GET("/")
1310         def _got_html(html):
1311             # These are radio buttons that allow the user to toggle
1312             # whether a particular mutable file is MDMF or SDMF.
1313             self.failUnlessIn("mutable-type-mdmf", html)
1314             self.failUnlessIn("mutable-type-sdmf", html)
1315             # We should also have the ability to create a mutable directory.
1316             self.failUnlessIn("mkdir", html)
1317             # ...and we should have the ability to say whether that's an
1318             # MDMF or SDMF directory
1319             self.failUnlessIn("mutable-directory-mdmf", html)
1320             self.failUnlessIn("mutable-directory-sdmf", html)
1321         d.addCallback(_got_html)
1322         return d
1323
1324     def test_mutable_type_defaults(self):
1325         # The checked="checked" attribute of the inputs corresponding to
1326         # the mutable-type parameter should change as expected with the
1327         # value configured in tahoe.cfg.
1328         #
1329         # By default, the value configured with the client is
1330         # SDMF_VERSION, so that should be checked.
1331         assert self.s.mutable_file_default == SDMF_VERSION
1332
1333         d = self.GET("/")
1334         def _got_html(html, value):
1335             i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1336             self.failUnlessIn(i % value, html)
1337         d.addCallback(_got_html, "sdmf")
1338         d.addCallback(lambda ignored:
1339             self.GET(self.public_url + "/foo", followRedirect=True))
1340         d.addCallback(_got_html, "sdmf")
1341         # Now switch the configuration value to MDMF. The MDMF radio
1342         # buttons should now be checked on these pages.
1343         def _swap_values(ignored):
1344             self.s.mutable_file_default = MDMF_VERSION
1345         d.addCallback(_swap_values)
1346         d.addCallback(lambda ignored: self.GET("/"))
1347         d.addCallback(_got_html, "mdmf")
1348         d.addCallback(lambda ignored:
1349             self.GET(self.public_url + "/foo", followRedirect=True))
1350         d.addCallback(_got_html, "mdmf")
1351         return d
1352
1353     def test_GET_DIRURL(self):
1354         # the addSlash means we get a redirect here
1355         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1356         ROOT = "../../.."
1357         d = self.GET(self.public_url + "/foo", followRedirect=True)
1358         def _check(res):
1359             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1360                             in res, res)
1361             # the FILE reference points to a URI, but it should end in bar.txt
1362             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1363                        (ROOT, urllib.quote(self._bar_txt_uri)))
1364             get_bar = "".join([r'<td>FILE</td>',
1365                                r'\s+<td>',
1366                                r'<a href="%s">bar.txt</a>' % bar_url,
1367                                r'</td>',
1368                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1369                                ])
1370             self.failUnless(re.search(get_bar, res), res)
1371             for label in ['unlink', 'rename']:
1372                 for line in res.split("\n"):
1373                     # find the line that contains the relevant button for bar.txt
1374                     if ("form action" in line and
1375                         ('value="%s"' % (label,)) in line and
1376                         'value="bar.txt"' in line):
1377                         # the form target should use a relative URL
1378                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1379                         self.failUnlessIn('action="%s"' % foo_url, line)
1380                         # and the when_done= should too
1381                         #done_url = urllib.quote(???)
1382                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1383
1384                         # 'unlink' needs to use POST because it directly has a side effect
1385                         if label == 'unlink':
1386                             self.failUnlessIn('method="post"', line)
1387                         break
1388                 else:
1389                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1390
1391             # the DIR reference just points to a URI
1392             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1393             get_sub = ((r'<td>DIR</td>')
1394                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1395             self.failUnless(re.search(get_sub, res), res)
1396         d.addCallback(_check)
1397
1398         # look at a readonly directory
1399         d.addCallback(lambda res:
1400                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1401         def _check2(res):
1402             self.failUnless("(read-only)" in res, res)
1403             self.failIf("Upload a file" in res, res)
1404         d.addCallback(_check2)
1405
1406         # and at a directory that contains a readonly directory
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url, followRedirect=True))
1409         def _check3(res):
1410             self.failUnless(re.search('<td>DIR-RO</td>'
1411                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1412         d.addCallback(_check3)
1413
1414         # and an empty directory
1415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1416         def _check4(res):
1417             self.failUnless("directory is empty" in res, res)
1418             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1419             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1420         d.addCallback(_check4)
1421
1422         # and at a literal directory
1423         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1424         d.addCallback(lambda res:
1425                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1426         def _check5(res):
1427             self.failUnless('(immutable)' in res, res)
1428             self.failUnless(re.search('<td>FILE</td>'
1429                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1430         d.addCallback(_check5)
1431         return d
1432
1433     def test_GET_DIRURL_badtype(self):
1434         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1435                                  400, "Bad Request",
1436                                  "bad t=bogus",
1437                                  self.GET,
1438                                  self.public_url + "/foo?t=bogus")
1439         return d
1440
1441     def test_GET_DIRURL_json(self):
1442         d = self.GET(self.public_url + "/foo?t=json")
1443         d.addCallback(self.failUnlessIsFooJSON)
1444         return d
1445
1446     def test_GET_DIRURL_json_format(self):
1447         d = self.PUT(self.public_url + \
1448                      "/foo/sdmf.txt?format=sdmf",
1449                      self.NEWFILE_CONTENTS * 300000)
1450         d.addCallback(lambda ignored:
1451             self.PUT(self.public_url + \
1452                      "/foo/mdmf.txt?format=mdmf",
1453                      self.NEWFILE_CONTENTS * 300000))
1454         # Now we have an MDMF and SDMF file in the directory. If we GET
1455         # its JSON, we should see their encodings.
1456         d.addCallback(lambda ignored:
1457             self.GET(self.public_url + "/foo?t=json"))
1458         def _got_json(json):
1459             data = simplejson.loads(json)
1460             assert data[0] == "dirnode"
1461
1462             data = data[1]
1463             kids = data['children']
1464
1465             mdmf_data = kids['mdmf.txt'][1]
1466             self.failUnlessIn("format", mdmf_data)
1467             self.failUnlessEqual(mdmf_data["format"], "mdmf")
1468
1469             sdmf_data = kids['sdmf.txt'][1]
1470             self.failUnlessIn("format", sdmf_data)
1471             self.failUnlessEqual(sdmf_data["format"], "sdmf")
1472         d.addCallback(_got_json)
1473         return d
1474
1475
1476     def test_POST_DIRURL_manifest_no_ophandle(self):
1477         d = self.shouldFail2(error.Error,
1478                              "test_POST_DIRURL_manifest_no_ophandle",
1479                              "400 Bad Request",
1480                              "slow operation requires ophandle=",
1481                              self.POST, self.public_url, t="start-manifest")
1482         return d
1483
1484     def test_POST_DIRURL_manifest(self):
1485         d = defer.succeed(None)
1486         def getman(ignored, output):
1487             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1488                           followRedirect=True)
1489             d.addCallback(self.wait_for_operation, "125")
1490             d.addCallback(self.get_operation_results, "125", output)
1491             return d
1492         d.addCallback(getman, None)
1493         def _got_html(manifest):
1494             self.failUnless("Manifest of SI=" in manifest)
1495             self.failUnless("<td>sub</td>" in manifest)
1496             self.failUnless(self._sub_uri in manifest)
1497             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1498         d.addCallback(_got_html)
1499
1500         # both t=status and unadorned GET should be identical
1501         d.addCallback(lambda res: self.GET("/operations/125"))
1502         d.addCallback(_got_html)
1503
1504         d.addCallback(getman, "html")
1505         d.addCallback(_got_html)
1506         d.addCallback(getman, "text")
1507         def _got_text(manifest):
1508             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1509             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1510         d.addCallback(_got_text)
1511         d.addCallback(getman, "JSON")
1512         def _got_json(res):
1513             data = res["manifest"]
1514             got = {}
1515             for (path_list, cap) in data:
1516                 got[tuple(path_list)] = cap
1517             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1518             self.failUnless((u"sub",u"baz.txt") in got)
1519             self.failUnless("finished" in res)
1520             self.failUnless("origin" in res)
1521             self.failUnless("storage-index" in res)
1522             self.failUnless("verifycaps" in res)
1523             self.failUnless("stats" in res)
1524         d.addCallback(_got_json)
1525         return d
1526
1527     def test_POST_DIRURL_deepsize_no_ophandle(self):
1528         d = self.shouldFail2(error.Error,
1529                              "test_POST_DIRURL_deepsize_no_ophandle",
1530                              "400 Bad Request",
1531                              "slow operation requires ophandle=",
1532                              self.POST, self.public_url, t="start-deep-size")
1533         return d
1534
1535     def test_POST_DIRURL_deepsize(self):
1536         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1537                       followRedirect=True)
1538         d.addCallback(self.wait_for_operation, "126")
1539         d.addCallback(self.get_operation_results, "126", "json")
1540         def _got_json(data):
1541             self.failUnlessReallyEqual(data["finished"], True)
1542             size = data["size"]
1543             self.failUnless(size > 1000)
1544         d.addCallback(_got_json)
1545         d.addCallback(self.get_operation_results, "126", "text")
1546         def _got_text(res):
1547             mo = re.search(r'^size: (\d+)$', res, re.M)
1548             self.failUnless(mo, res)
1549             size = int(mo.group(1))
1550             # with directories, the size varies.
1551             self.failUnless(size > 1000)
1552         d.addCallback(_got_text)
1553         return d
1554
1555     def test_POST_DIRURL_deepstats_no_ophandle(self):
1556         d = self.shouldFail2(error.Error,
1557                              "test_POST_DIRURL_deepstats_no_ophandle",
1558                              "400 Bad Request",
1559                              "slow operation requires ophandle=",
1560                              self.POST, self.public_url, t="start-deep-stats")
1561         return d
1562
1563     def test_POST_DIRURL_deepstats(self):
1564         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1565                       followRedirect=True)
1566         d.addCallback(self.wait_for_operation, "127")
1567         d.addCallback(self.get_operation_results, "127", "json")
1568         def _got_json(stats):
1569             expected = {"count-immutable-files": 3,
1570                         "count-mutable-files": 2,
1571                         "count-literal-files": 0,
1572                         "count-files": 5,
1573                         "count-directories": 3,
1574                         "size-immutable-files": 57,
1575                         "size-literal-files": 0,
1576                         #"size-directories": 1912, # varies
1577                         #"largest-directory": 1590,
1578                         "largest-directory-children": 7,
1579                         "largest-immutable-file": 19,
1580                         }
1581             for k,v in expected.iteritems():
1582                 self.failUnlessReallyEqual(stats[k], v,
1583                                            "stats[%s] was %s, not %s" %
1584                                            (k, stats[k], v))
1585             self.failUnlessReallyEqual(stats["size-files-histogram"],
1586                                        [ [11, 31, 3] ])
1587         d.addCallback(_got_json)
1588         return d
1589
1590     def test_POST_DIRURL_stream_manifest(self):
1591         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1592         def _check(res):
1593             self.failUnless(res.endswith("\n"))
1594             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1595             self.failUnlessReallyEqual(len(units), 9)
1596             self.failUnlessEqual(units[-1]["type"], "stats")
1597             first = units[0]
1598             self.failUnlessEqual(first["path"], [])
1599             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1600             self.failUnlessEqual(first["type"], "directory")
1601             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1602             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1603             self.failIfEqual(baz["storage-index"], None)
1604             self.failIfEqual(baz["verifycap"], None)
1605             self.failIfEqual(baz["repaircap"], None)
1606             # XXX: Add quux and baz to this test.
1607             return
1608         d.addCallback(_check)
1609         return d
1610
1611     def test_GET_DIRURL_uri(self):
1612         d = self.GET(self.public_url + "/foo?t=uri")
1613         def _check(res):
1614             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1615         d.addCallback(_check)
1616         return d
1617
1618     def test_GET_DIRURL_readonly_uri(self):
1619         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1620         def _check(res):
1621             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1622         d.addCallback(_check)
1623         return d
1624
1625     def test_PUT_NEWDIRURL(self):
1626         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, [])
1631         return d
1632
1633     def test_PUT_NEWDIRURL_mdmf(self):
1634         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1635         d.addCallback(lambda res:
1636                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(lambda node:
1639             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1640         return d
1641
1642     def test_PUT_NEWDIRURL_sdmf(self):
1643         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1644                      "")
1645         d.addCallback(lambda res:
1646                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648         d.addCallback(lambda node:
1649             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1650         return d
1651
1652     def test_PUT_NEWDIRURL_bad_format(self):
1653         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1654                                     400, "Bad Request", "Unknown format: foo",
1655                                     self.PUT, self.public_url +
1656                                     "/foo/newdir=?t=mkdir&format=foo", "")
1657
1658     def test_POST_NEWDIRURL(self):
1659         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1660         d.addCallback(lambda res:
1661                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1662         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1663         d.addCallback(self.failUnlessNodeKeysAre, [])
1664         return d
1665
1666     def test_POST_NEWDIRURL_mdmf(self):
1667         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1668         d.addCallback(lambda res:
1669                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1670         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1671         d.addCallback(lambda node:
1672             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1673         return d
1674
1675     def test_POST_NEWDIRURL_sdmf(self):
1676         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1677         d.addCallback(lambda res:
1678             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1679         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680         d.addCallback(lambda node:
1681             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1682         return d
1683
1684     def test_POST_NEWDIRURL_bad_format(self):
1685         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1686                                     400, "Bad Request", "Unknown format: foo",
1687                                     self.POST2, self.public_url + \
1688                                     "/foo/newdir?t=mkdir&format=foo", "")
1689
1690     def test_POST_NEWDIRURL_emptyname(self):
1691         # an empty pathname component (i.e. a double-slash) is disallowed
1692         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1693                              "400 Bad Request",
1694                              "The webapi does not allow empty pathname components, i.e. a double slash",
1695                              self.POST, self.public_url + "//?t=mkdir")
1696         return d
1697
1698     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1699         (newkids, caps) = self._create_initial_children()
1700         query = "/foo/newdir?t=mkdir-with-children"
1701         if version == MDMF_VERSION:
1702             query += "&format=mdmf"
1703         elif version == SDMF_VERSION:
1704             query += "&format=sdmf"
1705         else:
1706             version = SDMF_VERSION # for later
1707         d = self.POST2(self.public_url + query,
1708                        simplejson.dumps(newkids))
1709         def _check(uri):
1710             n = self.s.create_node_from_uri(uri.strip())
1711             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1712             self.failUnlessEqual(n._node.get_version(), version)
1713             d2.addCallback(lambda ign:
1714                            self.failUnlessROChildURIIs(n, u"child-imm",
1715                                                        caps['filecap1']))
1716             d2.addCallback(lambda ign:
1717                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1718                                                        caps['filecap2']))
1719             d2.addCallback(lambda ign:
1720                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1721                                                        caps['filecap3']))
1722             d2.addCallback(lambda ign:
1723                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1724                                                        caps['unknown_rocap']))
1725             d2.addCallback(lambda ign:
1726                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1727                                                        caps['unknown_rwcap']))
1728             d2.addCallback(lambda ign:
1729                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1730                                                        caps['unknown_immcap']))
1731             d2.addCallback(lambda ign:
1732                            self.failUnlessRWChildURIIs(n, u"dirchild",
1733                                                        caps['dircap']))
1734             d2.addCallback(lambda ign:
1735                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1736                                                        caps['litdircap']))
1737             d2.addCallback(lambda ign:
1738                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1739                                                        caps['emptydircap']))
1740             return d2
1741         d.addCallback(_check)
1742         d.addCallback(lambda res:
1743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1748         return d
1749
1750     def test_POST_NEWDIRURL_initial_children(self):
1751         return self._do_POST_NEWDIRURL_initial_children_test()
1752
1753     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1754         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1755
1756     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1757         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1758
1759     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1760         (newkids, caps) = self._create_initial_children()
1761         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1762                                     400, "Bad Request", "Unknown format: foo",
1763                                     self.POST2, self.public_url + \
1764                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1765                                     simplejson.dumps(newkids))
1766
1767     def test_POST_NEWDIRURL_immutable(self):
1768         (newkids, caps) = self._create_immutable_children()
1769         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1770                        simplejson.dumps(newkids))
1771         def _check(uri):
1772             n = self.s.create_node_from_uri(uri.strip())
1773             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1774             d2.addCallback(lambda ign:
1775                            self.failUnlessROChildURIIs(n, u"child-imm",
1776                                                        caps['filecap1']))
1777             d2.addCallback(lambda ign:
1778                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1779                                                        caps['unknown_immcap']))
1780             d2.addCallback(lambda ign:
1781                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1782                                                        caps['immdircap']))
1783             d2.addCallback(lambda ign:
1784                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1785                                                        caps['litdircap']))
1786             d2.addCallback(lambda ign:
1787                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1788                                                        caps['emptydircap']))
1789             return d2
1790         d.addCallback(_check)
1791         d.addCallback(lambda res:
1792                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1795         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1796         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1800         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1802         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1805         d.addErrback(self.explain_web_error)
1806         return d
1807
1808     def test_POST_NEWDIRURL_immutable_bad(self):
1809         (newkids, caps) = self._create_initial_children()
1810         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1811                              "400 Bad Request",
1812                              "needed to be immutable but was not",
1813                              self.POST2,
1814                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1815                              simplejson.dumps(newkids))
1816         return d
1817
1818     def test_PUT_NEWDIRURL_exists(self):
1819         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1822         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1823         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1824         return d
1825
1826     def test_PUT_NEWDIRURL_blocked(self):
1827         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1828                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1829                              self.PUT,
1830                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1833         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1834         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mkdir_p(self):
1838         d = defer.succeed(None)
1839         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1840         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1841         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1842         def mkdir_p(mkpnode):
1843             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1844             d = self.POST(url)
1845             def made_subsub(ssuri):
1846                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1847                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1848                 d = self.POST(url)
1849                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1850                 return d
1851             d.addCallback(made_subsub)
1852             return d
1853         d.addCallback(mkdir_p)
1854         return d
1855
1856     def test_PUT_NEWDIRURL_mkdirs(self):
1857         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1858         d.addCallback(lambda res:
1859                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1860         d.addCallback(lambda res:
1861                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1862         d.addCallback(lambda res:
1863                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1864         d.addCallback(self.failUnlessNodeKeysAre, [])
1865         return d
1866
1867     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1868         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1869         d.addCallback(lambda ignored:
1870             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1871         d.addCallback(lambda ignored:
1872             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1873         d.addCallback(lambda ignored:
1874             self._foo_node.get_child_at_path(u"subdir"))
1875         def _got_subdir(subdir):
1876             # XXX: What we want?
1877             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1878             self.failUnlessNodeHasChild(subdir, u"newdir")
1879             return subdir.get_child_at_path(u"newdir")
1880         d.addCallback(_got_subdir)
1881         d.addCallback(lambda newdir:
1882             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1883         return d
1884
1885     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1886         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1887         d.addCallback(lambda ignored:
1888             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1889         d.addCallback(lambda ignored:
1890             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1891         d.addCallback(lambda ignored:
1892             self._foo_node.get_child_at_path(u"subdir"))
1893         def _got_subdir(subdir):
1894             # XXX: What we want?
1895             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1896             self.failUnlessNodeHasChild(subdir, u"newdir")
1897             return subdir.get_child_at_path(u"newdir")
1898         d.addCallback(_got_subdir)
1899         d.addCallback(lambda newdir:
1900             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1901         return d
1902
1903     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1904         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1905                                     400, "Bad Request", "Unknown format: foo",
1906                                     self.PUT, self.public_url + \
1907                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1908                                     "")
1909
1910     def test_DELETE_DIRURL(self):
1911         d = self.DELETE(self.public_url + "/foo")
1912         d.addCallback(lambda res:
1913                       self.failIfNodeHasChild(self.public_root, u"foo"))
1914         return d
1915
1916     def test_DELETE_DIRURL_missing(self):
1917         d = self.DELETE(self.public_url + "/foo/missing")
1918         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1919         d.addCallback(lambda res:
1920                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1921         return d
1922
1923     def test_DELETE_DIRURL_missing2(self):
1924         d = self.DELETE(self.public_url + "/missing")
1925         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1926         return d
1927
1928     def dump_root(self):
1929         print "NODEWALK"
1930         w = webish.DirnodeWalkerMixin()
1931         def visitor(childpath, childnode, metadata):
1932             print childpath
1933         d = w.walk(self.public_root, visitor)
1934         return d
1935
1936     def failUnlessNodeKeysAre(self, node, expected_keys):
1937         for k in expected_keys:
1938             assert isinstance(k, unicode)
1939         d = node.list()
1940         def _check(children):
1941             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1942         d.addCallback(_check)
1943         return d
1944     def failUnlessNodeHasChild(self, node, name):
1945         assert isinstance(name, unicode)
1946         d = node.list()
1947         def _check(children):
1948             self.failUnless(name in children)
1949         d.addCallback(_check)
1950         return d
1951     def failIfNodeHasChild(self, node, name):
1952         assert isinstance(name, unicode)
1953         d = node.list()
1954         def _check(children):
1955             self.failIf(name in children)
1956         d.addCallback(_check)
1957         return d
1958
1959     def failUnlessChildContentsAre(self, node, name, expected_contents):
1960         assert isinstance(name, unicode)
1961         d = node.get_child_at_path(name)
1962         d.addCallback(lambda node: download_to_data(node))
1963         def _check(contents):
1964             self.failUnlessReallyEqual(contents, expected_contents)
1965         d.addCallback(_check)
1966         return d
1967
1968     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1969         assert isinstance(name, unicode)
1970         d = node.get_child_at_path(name)
1971         d.addCallback(lambda node: node.download_best_version())
1972         def _check(contents):
1973             self.failUnlessReallyEqual(contents, expected_contents)
1974         d.addCallback(_check)
1975         return d
1976
1977     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1978         assert isinstance(name, unicode)
1979         d = node.get_child_at_path(name)
1980         def _check(child):
1981             self.failUnless(child.is_unknown() or not child.is_readonly())
1982             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1983             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1984             expected_ro_uri = self._make_readonly(expected_uri)
1985             if expected_ro_uri:
1986                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1987         d.addCallback(_check)
1988         return d
1989
1990     def failUnlessROChildURIIs(self, node, name, expected_uri):
1991         assert isinstance(name, unicode)
1992         d = node.get_child_at_path(name)
1993         def _check(child):
1994             self.failUnless(child.is_unknown() or child.is_readonly())
1995             self.failUnlessReallyEqual(child.get_write_uri(), None)
1996             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1997             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1998         d.addCallback(_check)
1999         return d
2000
2001     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2002         assert isinstance(name, unicode)
2003         d = node.get_child_at_path(name)
2004         def _check(child):
2005             self.failUnless(child.is_unknown() or not child.is_readonly())
2006             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2007             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2008             expected_ro_uri = self._make_readonly(got_uri)
2009             if expected_ro_uri:
2010                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2011         d.addCallback(_check)
2012         return d
2013
2014     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2015         assert isinstance(name, unicode)
2016         d = node.get_child_at_path(name)
2017         def _check(child):
2018             self.failUnless(child.is_unknown() or child.is_readonly())
2019             self.failUnlessReallyEqual(child.get_write_uri(), None)
2020             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2021             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2022         d.addCallback(_check)
2023         return d
2024
2025     def failUnlessCHKURIHasContents(self, got_uri, contents):
2026         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2027
2028     def test_POST_upload(self):
2029         d = self.POST(self.public_url + "/foo", t="upload",
2030                       file=("new.txt", self.NEWFILE_CONTENTS))
2031         fn = self._foo_node
2032         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2033         d.addCallback(lambda res:
2034                       self.failUnlessChildContentsAre(fn, u"new.txt",
2035                                                       self.NEWFILE_CONTENTS))
2036         return d
2037
2038     def test_POST_upload_unicode(self):
2039         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2040         d = self.POST(self.public_url + "/foo", t="upload",
2041                       file=(filename, self.NEWFILE_CONTENTS))
2042         fn = self._foo_node
2043         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2044         d.addCallback(lambda res:
2045                       self.failUnlessChildContentsAre(fn, filename,
2046                                                       self.NEWFILE_CONTENTS))
2047         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2048         d.addCallback(lambda res: self.GET(target_url))
2049         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2050                                                                   self.NEWFILE_CONTENTS,
2051                                                                   contents))
2052         return d
2053
2054     def test_POST_upload_unicode_named(self):
2055         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2056         d = self.POST(self.public_url + "/foo", t="upload",
2057                       name=filename,
2058                       file=("overridden", self.NEWFILE_CONTENTS))
2059         fn = self._foo_node
2060         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2061         d.addCallback(lambda res:
2062                       self.failUnlessChildContentsAre(fn, filename,
2063                                                       self.NEWFILE_CONTENTS))
2064         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2065         d.addCallback(lambda res: self.GET(target_url))
2066         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2067                                                                   self.NEWFILE_CONTENTS,
2068                                                                   contents))
2069         return d
2070
2071     def test_POST_upload_no_link(self):
2072         d = self.POST("/uri", t="upload",
2073                       file=("new.txt", self.NEWFILE_CONTENTS))
2074         def _check_upload_results(page):
2075             # this should be a page which describes the results of the upload
2076             # that just finished.
2077             self.failUnless("Upload Results:" in page)
2078             self.failUnless("URI:" in page)
2079             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2080             mo = uri_re.search(page)
2081             self.failUnless(mo, page)
2082             new_uri = mo.group(1)
2083             return new_uri
2084         d.addCallback(_check_upload_results)
2085         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2086         return d
2087
2088     def test_POST_upload_no_link_whendone(self):
2089         d = self.POST("/uri", t="upload", when_done="/",
2090                       file=("new.txt", self.NEWFILE_CONTENTS))
2091         d.addBoth(self.shouldRedirect, "/")
2092         return d
2093
2094     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2095         d = defer.maybeDeferred(callable, *args, **kwargs)
2096         def done(res):
2097             if isinstance(res, failure.Failure):
2098                 res.trap(error.PageRedirect)
2099                 statuscode = res.value.status
2100                 target = res.value.location
2101                 return checker(statuscode, target)
2102             self.fail("%s: callable was supposed to redirect, not return '%s'"
2103                       % (which, res))
2104         d.addBoth(done)
2105         return d
2106
2107     def test_POST_upload_no_link_whendone_results(self):
2108         def check(statuscode, target):
2109             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2110             self.failUnless(target.startswith(self.webish_url), target)
2111             return client.getPage(target, method="GET")
2112         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2113                                  check,
2114                                  self.POST, "/uri", t="upload",
2115                                  when_done="/uri/%(uri)s",
2116                                  file=("new.txt", self.NEWFILE_CONTENTS))
2117         d.addCallback(lambda res:
2118                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2119         return d
2120
2121     def test_POST_upload_no_link_mutable(self):
2122         d = self.POST("/uri", t="upload", mutable="true",
2123                       file=("new.txt", self.NEWFILE_CONTENTS))
2124         def _check(filecap):
2125             filecap = filecap.strip()
2126             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2127             self.filecap = filecap
2128             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2129             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2130             n = self.s.create_node_from_uri(filecap)
2131             return n.download_best_version()
2132         d.addCallback(_check)
2133         def _check2(data):
2134             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2135             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2136         d.addCallback(_check2)
2137         def _check3(data):
2138             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2139             return self.GET("/file/%s" % urllib.quote(self.filecap))
2140         d.addCallback(_check3)
2141         def _check4(data):
2142             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2143         d.addCallback(_check4)
2144         return d
2145
2146     def test_POST_upload_no_link_mutable_toobig(self):
2147         # The SDMF size limit is no longer in place, so we should be
2148         # able to upload mutable files that are as large as we want them
2149         # to be.
2150         d = self.POST("/uri", t="upload", mutable="true",
2151                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2152         return d
2153
2154
2155     def test_POST_upload_format_unlinked(self):
2156         def _check_upload_unlinked(ign, format, uri_prefix):
2157             filename = format + ".txt"
2158             d = self.POST("/uri?t=upload&format=" + format,
2159                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2160             def _got_filecap(filecap):
2161                 self.failUnless(filecap.startswith(uri_prefix))
2162                 return self.GET("/uri/%s?t=json" % filecap)
2163             d.addCallback(_got_filecap)
2164             def _got_json(json):
2165                 data = simplejson.loads(json)
2166                 data = data[1]
2167                 self.failUnlessIn("format", data)
2168                 self.failUnlessEqual(data["format"], format)
2169             d.addCallback(_got_json)
2170
2171         d = defer.succeed(None)
2172         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2173         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2174         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2175         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2176         return d
2177
2178     def test_POST_upload_bad_format_unlinked(self):
2179         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2180                                     400, "Bad Request", "Unknown format: foo",
2181                                     self.POST,
2182                                     "/uri?t=upload&format=foo",
2183                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2184
2185     def test_POST_upload_format(self):
2186         def _check_upload(ign, format, uri_prefix, fn=None):
2187             filename = format + ".txt"
2188             d = self.POST(self.public_url +
2189                           "/foo?t=upload&format=" + format,
2190                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2191             def _got_filecap(filecap):
2192                 if fn is not None:
2193                     filenameu = unicode(filename)
2194                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2195                 self.failUnless(filecap.startswith(uri_prefix))
2196                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2197             d.addCallback(_got_filecap)
2198             def _got_json(json):
2199                 data = simplejson.loads(json)
2200                 data = data[1]
2201                 self.failUnlessIn("format", data)
2202                 self.failUnlessEqual(data["format"], format)
2203             d.addCallback(_got_json)
2204
2205         d = defer.succeed(None)
2206         d.addCallback(_check_upload, "chk", "URI:CHK")
2207         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2208         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2209         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2210         return d
2211
2212     def test_POST_upload_bad_format(self):
2213         return self.shouldHTTPError("POST_upload_bad_format",
2214                                     400, "Bad Request", "Unknown format: foo",
2215                                     self.POST, self.public_url + \
2216                                     "/foo?t=upload&format=foo",
2217                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2218
2219     def test_POST_upload_mutable(self):
2220         # this creates a mutable file
2221         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2222                       file=("new.txt", self.NEWFILE_CONTENTS))
2223         fn = self._foo_node
2224         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2225         d.addCallback(lambda res:
2226                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2227                                                              self.NEWFILE_CONTENTS))
2228         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2229         def _got(newnode):
2230             self.failUnless(IMutableFileNode.providedBy(newnode))
2231             self.failUnless(newnode.is_mutable())
2232             self.failIf(newnode.is_readonly())
2233             self._mutable_node = newnode
2234             self._mutable_uri = newnode.get_uri()
2235         d.addCallback(_got)
2236
2237         # now upload it again and make sure that the URI doesn't change
2238         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2239         d.addCallback(lambda res:
2240                       self.POST(self.public_url + "/foo", t="upload",
2241                                 mutable="true",
2242                                 file=("new.txt", NEWER_CONTENTS)))
2243         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2244         d.addCallback(lambda res:
2245                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2246                                                              NEWER_CONTENTS))
2247         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2248         def _got2(newnode):
2249             self.failUnless(IMutableFileNode.providedBy(newnode))
2250             self.failUnless(newnode.is_mutable())
2251             self.failIf(newnode.is_readonly())
2252             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2253         d.addCallback(_got2)
2254
2255         # upload a second time, using PUT instead of POST
2256         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2257         d.addCallback(lambda res:
2258                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2259         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2260         d.addCallback(lambda res:
2261                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2262                                                              NEW2_CONTENTS))
2263
2264         # finally list the directory, since mutable files are displayed
2265         # slightly differently
2266
2267         d.addCallback(lambda res:
2268                       self.GET(self.public_url + "/foo/",
2269                                followRedirect=True))
2270         def _check_page(res):
2271             # TODO: assert more about the contents
2272             self.failUnless("SSK" in res)
2273             return res
2274         d.addCallback(_check_page)
2275
2276         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2277         def _got3(newnode):
2278             self.failUnless(IMutableFileNode.providedBy(newnode))
2279             self.failUnless(newnode.is_mutable())
2280             self.failIf(newnode.is_readonly())
2281             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2282         d.addCallback(_got3)
2283
2284         # look at the JSON form of the enclosing directory
2285         d.addCallback(lambda res:
2286                       self.GET(self.public_url + "/foo/?t=json",
2287                                followRedirect=True))
2288         def _check_page_json(res):
2289             parsed = simplejson.loads(res)
2290             self.failUnlessEqual(parsed[0], "dirnode")
2291             children = dict( [(unicode(name),value)
2292                               for (name,value)
2293                               in parsed[1]["children"].iteritems()] )
2294             self.failUnless(u"new.txt" in children)
2295             new_json = children[u"new.txt"]
2296             self.failUnlessEqual(new_json[0], "filenode")
2297             self.failUnless(new_json[1]["mutable"])
2298             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2299             ro_uri = self._mutable_node.get_readonly().to_string()
2300             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2301         d.addCallback(_check_page_json)
2302
2303         # and the JSON form of the file
2304         d.addCallback(lambda res:
2305                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2306         def _check_file_json(res):
2307             parsed = simplejson.loads(res)
2308             self.failUnlessEqual(parsed[0], "filenode")
2309             self.failUnless(parsed[1]["mutable"])
2310             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2311             ro_uri = self._mutable_node.get_readonly().to_string()
2312             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2313         d.addCallback(_check_file_json)
2314
2315         # and look at t=uri and t=readonly-uri
2316         d.addCallback(lambda res:
2317                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2318         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2319         d.addCallback(lambda res:
2320                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2321         def _check_ro_uri(res):
2322             ro_uri = self._mutable_node.get_readonly().to_string()
2323             self.failUnlessReallyEqual(res, ro_uri)
2324         d.addCallback(_check_ro_uri)
2325
2326         # make sure we can get to it from /uri/URI
2327         d.addCallback(lambda res:
2328                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2329         d.addCallback(lambda res:
2330                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2331
2332         # and that HEAD computes the size correctly
2333         d.addCallback(lambda res:
2334                       self.HEAD(self.public_url + "/foo/new.txt",
2335                                 return_response=True))
2336         def _got_headers((res, status, headers)):
2337             self.failUnlessReallyEqual(res, "")
2338             self.failUnlessReallyEqual(headers["content-length"][0],
2339                                        str(len(NEW2_CONTENTS)))
2340             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2341         d.addCallback(_got_headers)
2342
2343         # make sure that outdated size limits aren't enforced anymore.
2344         d.addCallback(lambda ignored:
2345             self.POST(self.public_url + "/foo", t="upload",
2346                       mutable="true",
2347                       file=("new.txt",
2348                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2349         d.addErrback(self.dump_error)
2350         return d
2351
2352     def test_POST_upload_mutable_toobig(self):
2353         # SDMF had a size limti that was removed a while ago. MDMF has
2354         # never had a size limit. Test to make sure that we do not
2355         # encounter errors when trying to upload large mutable files,
2356         # since there should be no coded prohibitions regarding large
2357         # mutable files.
2358         d = self.POST(self.public_url + "/foo",
2359                       t="upload", mutable="true",
2360                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2361         return d
2362
2363     def dump_error(self, f):
2364         # if the web server returns an error code (like 400 Bad Request),
2365         # web.client.getPage puts the HTTP response body into the .response
2366         # attribute of the exception object that it gives back. It does not
2367         # appear in the Failure's repr(), so the ERROR that trial displays
2368         # will be rather terse and unhelpful. addErrback this method to the
2369         # end of your chain to get more information out of these errors.
2370         if f.check(error.Error):
2371             print "web.error.Error:"
2372             print f
2373             print f.value.response
2374         return f
2375
2376     def test_POST_upload_replace(self):
2377         d = self.POST(self.public_url + "/foo", t="upload",
2378                       file=("bar.txt", self.NEWFILE_CONTENTS))
2379         fn = self._foo_node
2380         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2381         d.addCallback(lambda res:
2382                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2383                                                       self.NEWFILE_CONTENTS))
2384         return d
2385
2386     def test_POST_upload_no_replace_ok(self):
2387         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2388                       file=("new.txt", self.NEWFILE_CONTENTS))
2389         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2390         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2391                                                              self.NEWFILE_CONTENTS))
2392         return d
2393
2394     def test_POST_upload_no_replace_queryarg(self):
2395         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2396                       file=("bar.txt", self.NEWFILE_CONTENTS))
2397         d.addBoth(self.shouldFail, error.Error,
2398                   "POST_upload_no_replace_queryarg",
2399                   "409 Conflict",
2400                   "There was already a child by that name, and you asked me "
2401                   "to not replace it")
2402         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2403         d.addCallback(self.failUnlessIsBarDotTxt)
2404         return d
2405
2406     def test_POST_upload_no_replace_field(self):
2407         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2408                       file=("bar.txt", self.NEWFILE_CONTENTS))
2409         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2410                   "409 Conflict",
2411                   "There was already a child by that name, and you asked me "
2412                   "to not replace it")
2413         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2414         d.addCallback(self.failUnlessIsBarDotTxt)
2415         return d
2416
2417     def test_POST_upload_whendone(self):
2418         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2419                       file=("new.txt", self.NEWFILE_CONTENTS))
2420         d.addBoth(self.shouldRedirect, "/THERE")
2421         fn = self._foo_node
2422         d.addCallback(lambda res:
2423                       self.failUnlessChildContentsAre(fn, u"new.txt",
2424                                                       self.NEWFILE_CONTENTS))
2425         return d
2426
2427     def test_POST_upload_named(self):
2428         fn = self._foo_node
2429         d = self.POST(self.public_url + "/foo", t="upload",
2430                       name="new.txt", file=self.NEWFILE_CONTENTS)
2431         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2432         d.addCallback(lambda res:
2433                       self.failUnlessChildContentsAre(fn, u"new.txt",
2434                                                       self.NEWFILE_CONTENTS))
2435         return d
2436
2437     def test_POST_upload_named_badfilename(self):
2438         d = self.POST(self.public_url + "/foo", t="upload",
2439                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2440         d.addBoth(self.shouldFail, error.Error,
2441                   "test_POST_upload_named_badfilename",
2442                   "400 Bad Request",
2443                   "name= may not contain a slash",
2444                   )
2445         # make sure that nothing was added
2446         d.addCallback(lambda res:
2447                       self.failUnlessNodeKeysAre(self._foo_node,
2448                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2449                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2450                                                   u"sub"]))
2451         return d
2452
2453     def test_POST_FILEURL_check(self):
2454         bar_url = self.public_url + "/foo/bar.txt"
2455         d = self.POST(bar_url, t="check")
2456         def _check(res):
2457             self.failUnless("Healthy :" in res)
2458         d.addCallback(_check)
2459         redir_url = "http://allmydata.org/TARGET"
2460         def _check2(statuscode, target):
2461             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2462             self.failUnlessReallyEqual(target, redir_url)
2463         d.addCallback(lambda res:
2464                       self.shouldRedirect2("test_POST_FILEURL_check",
2465                                            _check2,
2466                                            self.POST, bar_url,
2467                                            t="check",
2468                                            when_done=redir_url))
2469         d.addCallback(lambda res:
2470                       self.POST(bar_url, t="check", return_to=redir_url))
2471         def _check3(res):
2472             self.failUnless("Healthy :" in res)
2473             self.failUnless("Return to file" in res)
2474             self.failUnless(redir_url in res)
2475         d.addCallback(_check3)
2476
2477         d.addCallback(lambda res:
2478                       self.POST(bar_url, t="check", output="JSON"))
2479         def _check_json(res):
2480             data = simplejson.loads(res)
2481             self.failUnless("storage-index" in data)
2482             self.failUnless(data["results"]["healthy"])
2483         d.addCallback(_check_json)
2484
2485         return d
2486
2487     def test_POST_FILEURL_check_and_repair(self):
2488         bar_url = self.public_url + "/foo/bar.txt"
2489         d = self.POST(bar_url, t="check", repair="true")
2490         def _check(res):
2491             self.failUnless("Healthy :" in res)
2492         d.addCallback(_check)
2493         redir_url = "http://allmydata.org/TARGET"
2494         def _check2(statuscode, target):
2495             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2496             self.failUnlessReallyEqual(target, redir_url)
2497         d.addCallback(lambda res:
2498                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2499                                            _check2,
2500                                            self.POST, bar_url,
2501                                            t="check", repair="true",
2502                                            when_done=redir_url))
2503         d.addCallback(lambda res:
2504                       self.POST(bar_url, t="check", return_to=redir_url))
2505         def _check3(res):
2506             self.failUnless("Healthy :" in res)
2507             self.failUnless("Return to file" in res)
2508             self.failUnless(redir_url in res)
2509         d.addCallback(_check3)
2510         return d
2511
2512     def test_POST_DIRURL_check(self):
2513         foo_url = self.public_url + "/foo/"
2514         d = self.POST(foo_url, t="check")
2515         def _check(res):
2516             self.failUnless("Healthy :" in res, res)
2517         d.addCallback(_check)
2518         redir_url = "http://allmydata.org/TARGET"
2519         def _check2(statuscode, target):
2520             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2521             self.failUnlessReallyEqual(target, redir_url)
2522         d.addCallback(lambda res:
2523                       self.shouldRedirect2("test_POST_DIRURL_check",
2524                                            _check2,
2525                                            self.POST, foo_url,
2526                                            t="check",
2527                                            when_done=redir_url))
2528         d.addCallback(lambda res:
2529                       self.POST(foo_url, t="check", return_to=redir_url))
2530         def _check3(res):
2531             self.failUnless("Healthy :" in res, res)
2532             self.failUnless("Return to file/directory" in res)
2533             self.failUnless(redir_url in res)
2534         d.addCallback(_check3)
2535
2536         d.addCallback(lambda res:
2537                       self.POST(foo_url, t="check", output="JSON"))
2538         def _check_json(res):
2539             data = simplejson.loads(res)
2540             self.failUnless("storage-index" in data)
2541             self.failUnless(data["results"]["healthy"])
2542         d.addCallback(_check_json)
2543
2544         return d
2545
2546     def test_POST_DIRURL_check_and_repair(self):
2547         foo_url = self.public_url + "/foo/"
2548         d = self.POST(foo_url, t="check", repair="true")
2549         def _check(res):
2550             self.failUnless("Healthy :" in res, res)
2551         d.addCallback(_check)
2552         redir_url = "http://allmydata.org/TARGET"
2553         def _check2(statuscode, target):
2554             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2555             self.failUnlessReallyEqual(target, redir_url)
2556         d.addCallback(lambda res:
2557                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2558                                            _check2,
2559                                            self.POST, foo_url,
2560                                            t="check", repair="true",
2561                                            when_done=redir_url))
2562         d.addCallback(lambda res:
2563                       self.POST(foo_url, t="check", return_to=redir_url))
2564         def _check3(res):
2565             self.failUnless("Healthy :" in res)
2566             self.failUnless("Return to file/directory" in res)
2567             self.failUnless(redir_url in res)
2568         d.addCallback(_check3)
2569         return d
2570
2571     def test_POST_FILEURL_mdmf_check(self):
2572         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2573         d = self.POST(quux_url, t="check")
2574         def _check(res):
2575             self.failUnlessIn("Healthy", res)
2576         d.addCallback(_check)
2577         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2578         d.addCallback(lambda ignored:
2579             self.POST(quux_extension_url, t="check"))
2580         d.addCallback(_check)
2581         return d
2582
2583     def test_POST_FILEURL_mdmf_check_and_repair(self):
2584         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2585         d = self.POST(quux_url, t="check", repair="true")
2586         def _check(res):
2587             self.failUnlessIn("Healthy", res)
2588         d.addCallback(_check)
2589         quux_extension_url = "/uri/%s" %\
2590             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2591         d.addCallback(lambda ignored:
2592             self.POST(quux_extension_url, t="check", repair="true"))
2593         d.addCallback(_check)
2594         return d
2595
2596     def wait_for_operation(self, ignored, ophandle):
2597         url = "/operations/" + ophandle
2598         url += "?t=status&output=JSON"
2599         d = self.GET(url)
2600         def _got(res):
2601             data = simplejson.loads(res)
2602             if not data["finished"]:
2603                 d = self.stall(delay=1.0)
2604                 d.addCallback(self.wait_for_operation, ophandle)
2605                 return d
2606             return data
2607         d.addCallback(_got)
2608         return d
2609
2610     def get_operation_results(self, ignored, ophandle, output=None):
2611         url = "/operations/" + ophandle
2612         url += "?t=status"
2613         if output:
2614             url += "&output=" + output
2615         d = self.GET(url)
2616         def _got(res):
2617             if output and output.lower() == "json":
2618                 return simplejson.loads(res)
2619             return res
2620         d.addCallback(_got)
2621         return d
2622
2623     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2624         d = self.shouldFail2(error.Error,
2625                              "test_POST_DIRURL_deepcheck_no_ophandle",
2626                              "400 Bad Request",
2627                              "slow operation requires ophandle=",
2628                              self.POST, self.public_url, t="start-deep-check")
2629         return d
2630
2631     def test_POST_DIRURL_deepcheck(self):
2632         def _check_redirect(statuscode, target):
2633             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2634             self.failUnless(target.endswith("/operations/123"))
2635         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2636                                  self.POST, self.public_url,
2637                                  t="start-deep-check", ophandle="123")
2638         d.addCallback(self.wait_for_operation, "123")
2639         def _check_json(data):
2640             self.failUnlessReallyEqual(data["finished"], True)
2641             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2642             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2643         d.addCallback(_check_json)
2644         d.addCallback(self.get_operation_results, "123", "html")
2645         def _check_html(res):
2646             self.failUnless("Objects Checked: <span>10</span>" in res)
2647             self.failUnless("Objects Healthy: <span>10</span>" in res)
2648         d.addCallback(_check_html)
2649
2650         d.addCallback(lambda res:
2651                       self.GET("/operations/123/"))
2652         d.addCallback(_check_html) # should be the same as without the slash
2653
2654         d.addCallback(lambda res:
2655                       self.shouldFail2(error.Error, "one", "404 Not Found",
2656                                        "No detailed results for SI bogus",
2657                                        self.GET, "/operations/123/bogus"))
2658
2659         foo_si = self._foo_node.get_storage_index()
2660         foo_si_s = base32.b2a(foo_si)
2661         d.addCallback(lambda res:
2662                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2663         def _check_foo_json(res):
2664             data = simplejson.loads(res)
2665             self.failUnlessEqual(data["storage-index"], foo_si_s)
2666             self.failUnless(data["results"]["healthy"])
2667         d.addCallback(_check_foo_json)
2668         return d
2669
2670     def test_POST_DIRURL_deepcheck_and_repair(self):
2671         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2672                       ophandle="124", output="json", followRedirect=True)
2673         d.addCallback(self.wait_for_operation, "124")
2674         def _check_json(data):
2675             self.failUnlessReallyEqual(data["finished"], True)
2676             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2677             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2678             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2679             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2680             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2681             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2682             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2683             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2684             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2685             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2686         d.addCallback(_check_json)
2687         d.addCallback(self.get_operation_results, "124", "html")
2688         def _check_html(res):
2689             self.failUnless("Objects Checked: <span>10</span>" in res)
2690
2691             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2692             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2693             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2694
2695             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2696             self.failUnless("Repairs Successful: <span>0</span>" in res)
2697             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2698
2699             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2700             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2701             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2702         d.addCallback(_check_html)
2703         return d
2704
2705     def test_POST_FILEURL_bad_t(self):
2706         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2707                              "POST to file: bad t=bogus",
2708                              self.POST, self.public_url + "/foo/bar.txt",
2709                              t="bogus")
2710         return d
2711
2712     def test_POST_mkdir(self): # return value?
2713         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(self.failUnlessNodeKeysAre, [])
2716         return d
2717
2718     def test_POST_mkdir_mdmf(self):
2719         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2720         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2721         d.addCallback(lambda node:
2722             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2723         return d
2724
2725     def test_POST_mkdir_sdmf(self):
2726         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2727         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2728         d.addCallback(lambda node:
2729             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2730         return d
2731
2732     def test_POST_mkdir_bad_format(self):
2733         return self.shouldHTTPError("POST_mkdir_bad_format",
2734                                     400, "Bad Request", "Unknown format: foo",
2735                                     self.POST, self.public_url +
2736                                     "/foo?t=mkdir&name=newdir&format=foo")
2737
2738     def test_POST_mkdir_initial_children(self):
2739         (newkids, caps) = self._create_initial_children()
2740         d = self.POST2(self.public_url +
2741                        "/foo?t=mkdir-with-children&name=newdir",
2742                        simplejson.dumps(newkids))
2743         d.addCallback(lambda res:
2744                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2745         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2746         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2747         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2748         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2749         return d
2750
2751     def test_POST_mkdir_initial_children_mdmf(self):
2752         (newkids, caps) = self._create_initial_children()
2753         d = self.POST2(self.public_url +
2754                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2755                        simplejson.dumps(newkids))
2756         d.addCallback(lambda res:
2757                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2758         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2759         d.addCallback(lambda node:
2760             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2762         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2763                        caps['filecap1'])
2764         return d
2765
2766     # XXX: Duplication.
2767     def test_POST_mkdir_initial_children_sdmf(self):
2768         (newkids, caps) = self._create_initial_children()
2769         d = self.POST2(self.public_url +
2770                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2771                        simplejson.dumps(newkids))
2772         d.addCallback(lambda res:
2773                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2774         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2775         d.addCallback(lambda node:
2776             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2777         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2778         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2779                        caps['filecap1'])
2780         return d
2781
2782     def test_POST_mkdir_initial_children_bad_format(self):
2783         (newkids, caps) = self._create_initial_children()
2784         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2785                                     400, "Bad Request", "Unknown format: foo",
2786                                     self.POST, self.public_url + \
2787                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2788                                     simplejson.dumps(newkids))
2789
2790     def test_POST_mkdir_immutable(self):
2791         (newkids, caps) = self._create_immutable_children()
2792         d = self.POST2(self.public_url +
2793                        "/foo?t=mkdir-immutable&name=newdir",
2794                        simplejson.dumps(newkids))
2795         d.addCallback(lambda res:
2796                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2798         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2800         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2802         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2805         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2806         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2807         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2808         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2809         return d
2810
2811     def test_POST_mkdir_immutable_bad(self):
2812         (newkids, caps) = self._create_initial_children()
2813         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2814                              "400 Bad Request",
2815                              "needed to be immutable but was not",
2816                              self.POST2,
2817                              self.public_url +
2818                              "/foo?t=mkdir-immutable&name=newdir",
2819                              simplejson.dumps(newkids))
2820         return d
2821
2822     def test_POST_mkdir_2(self):
2823         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2824         d.addCallback(lambda res:
2825                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2826         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2827         d.addCallback(self.failUnlessNodeKeysAre, [])
2828         return d
2829
2830     def test_POST_mkdirs_2(self):
2831         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2832         d.addCallback(lambda res:
2833                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2834         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2835         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2836         d.addCallback(self.failUnlessNodeKeysAre, [])
2837         return d
2838
2839     def test_POST_mkdir_no_parentdir_noredirect(self):
2840         d = self.POST("/uri?t=mkdir")
2841         def _after_mkdir(res):
2842             uri.DirectoryURI.init_from_string(res)
2843         d.addCallback(_after_mkdir)
2844         return d
2845
2846     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2847         d = self.POST("/uri?t=mkdir&format=mdmf")
2848         def _after_mkdir(res):
2849             u = uri.from_string(res)
2850             # Check that this is an MDMF writecap
2851             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2852         d.addCallback(_after_mkdir)
2853         return d
2854
2855     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2856         d = self.POST("/uri?t=mkdir&format=sdmf")
2857         def _after_mkdir(res):
2858             u = uri.from_string(res)
2859             self.failUnlessIsInstance(u, uri.DirectoryURI)
2860         d.addCallback(_after_mkdir)
2861         return d
2862
2863     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2864         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2865                                     400, "Bad Request", "Unknown format: foo",
2866                                     self.POST, self.public_url +
2867                                     "/uri?t=mkdir&format=foo")
2868
2869     def test_POST_mkdir_no_parentdir_noredirect2(self):
2870         # make sure form-based arguments (as on the welcome page) still work
2871         d = self.POST("/uri", t="mkdir")
2872         def _after_mkdir(res):
2873             uri.DirectoryURI.init_from_string(res)
2874         d.addCallback(_after_mkdir)
2875         d.addErrback(self.explain_web_error)
2876         return d
2877
2878     def test_POST_mkdir_no_parentdir_redirect(self):
2879         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2880         d.addBoth(self.shouldRedirect, None, statuscode='303')
2881         def _check_target(target):
2882             target = urllib.unquote(target)
2883             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2884         d.addCallback(_check_target)
2885         return d
2886
2887     def test_POST_mkdir_no_parentdir_redirect2(self):
2888         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2889         d.addBoth(self.shouldRedirect, None, statuscode='303')
2890         def _check_target(target):
2891             target = urllib.unquote(target)
2892             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2893         d.addCallback(_check_target)
2894         d.addErrback(self.explain_web_error)
2895         return d
2896
2897     def _make_readonly(self, u):
2898         ro_uri = uri.from_string(u).get_readonly()
2899         if ro_uri is None:
2900             return None
2901         return ro_uri.to_string()
2902
2903     def _create_initial_children(self):
2904         contents, n, filecap1 = self.makefile(12)
2905         md1 = {"metakey1": "metavalue1"}
2906         filecap2 = make_mutable_file_uri()
2907         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2908         filecap3 = node3.get_readonly_uri()
2909         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2910         dircap = DirectoryNode(node4, None, None).get_uri()
2911         mdmfcap = make_mutable_file_uri(mdmf=True)
2912         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2913         emptydircap = "URI:DIR2-LIT:"
2914         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2915                                                       "ro_uri": self._make_readonly(filecap1),
2916                                                       "metadata": md1, }],
2917                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2918                                                       "ro_uri": self._make_readonly(filecap2)}],
2919                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2920                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2921                                                       "ro_uri": unknown_rocap}],
2922                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2923                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2924                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2925                                                       "ro_uri": self._make_readonly(dircap)}],
2926                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2927                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2928                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2929                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2930                    }
2931         return newkids, {'filecap1': filecap1,
2932                          'filecap2': filecap2,
2933                          'filecap3': filecap3,
2934                          'unknown_rwcap': unknown_rwcap,
2935                          'unknown_rocap': unknown_rocap,
2936                          'unknown_immcap': unknown_immcap,
2937                          'dircap': dircap,
2938                          'litdircap': litdircap,
2939                          'emptydircap': emptydircap,
2940                          'mdmfcap': mdmfcap}
2941
2942     def _create_immutable_children(self):
2943         contents, n, filecap1 = self.makefile(12)
2944         md1 = {"metakey1": "metavalue1"}
2945         tnode = create_chk_filenode("immutable directory contents\n"*10)
2946         dnode = DirectoryNode(tnode, None, None)
2947         assert not dnode.is_mutable()
2948         immdircap = dnode.get_uri()
2949         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2950         emptydircap = "URI:DIR2-LIT:"
2951         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2952                                                       "metadata": md1, }],
2953                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2954                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2955                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2956                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2957                    }
2958         return newkids, {'filecap1': filecap1,
2959                          'unknown_immcap': unknown_immcap,
2960                          'immdircap': immdircap,
2961                          'litdircap': litdircap,
2962                          'emptydircap': emptydircap}
2963
2964     def test_POST_mkdir_no_parentdir_initial_children(self):
2965         (newkids, caps) = self._create_initial_children()
2966         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2967         def _after_mkdir(res):
2968             self.failUnless(res.startswith("URI:DIR"), res)
2969             n = self.s.create_node_from_uri(res)
2970             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2971             d2.addCallback(lambda ign:
2972                            self.failUnlessROChildURIIs(n, u"child-imm",
2973                                                        caps['filecap1']))
2974             d2.addCallback(lambda ign:
2975                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2976                                                        caps['filecap2']))
2977             d2.addCallback(lambda ign:
2978                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2979                                                        caps['filecap3']))
2980             d2.addCallback(lambda ign:
2981                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2982                                                        caps['unknown_rwcap']))
2983             d2.addCallback(lambda ign:
2984                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2985                                                        caps['unknown_rocap']))
2986             d2.addCallback(lambda ign:
2987                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2988                                                        caps['unknown_immcap']))
2989             d2.addCallback(lambda ign:
2990                            self.failUnlessRWChildURIIs(n, u"dirchild",
2991                                                        caps['dircap']))
2992             return d2
2993         d.addCallback(_after_mkdir)
2994         return d
2995
2996     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2997         # the regular /uri?t=mkdir operation is specified to ignore its body.
2998         # Only t=mkdir-with-children pays attention to it.
2999         (newkids, caps) = self._create_initial_children()
3000         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3001                                  400, "Bad Request",
3002                                  "t=mkdir does not accept children=, "
3003                                  "try t=mkdir-with-children instead",
3004                                  self.POST2, "/uri?t=mkdir", # without children
3005                                  simplejson.dumps(newkids))
3006         return d
3007
3008     def test_POST_noparent_bad(self):
3009         d = self.shouldHTTPError("POST_noparent_bad",
3010                                  400, "Bad Request",
3011                                  "/uri accepts only PUT, PUT?t=mkdir, "
3012                                  "POST?t=upload, and POST?t=mkdir",
3013                                  self.POST, "/uri?t=bogus")
3014         return d
3015
3016     def test_POST_mkdir_no_parentdir_immutable(self):
3017         (newkids, caps) = self._create_immutable_children()
3018         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3019         def _after_mkdir(res):
3020             self.failUnless(res.startswith("URI:DIR"), res)
3021             n = self.s.create_node_from_uri(res)
3022             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3023             d2.addCallback(lambda ign:
3024                            self.failUnlessROChildURIIs(n, u"child-imm",
3025                                                           caps['filecap1']))
3026             d2.addCallback(lambda ign:
3027                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3028                                                           caps['unknown_immcap']))
3029             d2.addCallback(lambda ign:
3030                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3031                                                           caps['immdircap']))
3032             d2.addCallback(lambda ign:
3033                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3034                                                           caps['litdircap']))
3035             d2.addCallback(lambda ign:
3036                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3037                                                           caps['emptydircap']))
3038             return d2
3039         d.addCallback(_after_mkdir)
3040         return d
3041
3042     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3043         (newkids, caps) = self._create_initial_children()
3044         d = self.shouldFail2(error.Error,
3045                              "test_POST_mkdir_no_parentdir_immutable_bad",
3046                              "400 Bad Request",
3047                              "needed to be immutable but was not",
3048                              self.POST2,
3049                              "/uri?t=mkdir-immutable",
3050                              simplejson.dumps(newkids))
3051         return d
3052
3053     def test_welcome_page_mkdir_button(self):
3054         # Fetch the welcome page.
3055         d = self.GET("/")
3056         def _after_get_welcome_page(res):
3057             MKDIR_BUTTON_RE = re.compile(
3058                 '<form action="([^"]*)" method="post".*?'
3059                 '<input type="hidden" name="t" value="([^"]*)" />'
3060                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3061                 '<input type="submit" value="Create a directory" />',
3062                 re.I)
3063             mo = MKDIR_BUTTON_RE.search(res)
3064             formaction = mo.group(1)
3065             formt = mo.group(2)
3066             formaname = mo.group(3)
3067             formavalue = mo.group(4)
3068             return (formaction, formt, formaname, formavalue)
3069         d.addCallback(_after_get_welcome_page)
3070         def _after_parse_form(res):
3071             (formaction, formt, formaname, formavalue) = res
3072             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3073         d.addCallback(_after_parse_form)
3074         d.addBoth(self.shouldRedirect, None, statuscode='303')
3075         return d
3076
3077     def test_POST_mkdir_replace(self): # return value?
3078         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3079         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3080         d.addCallback(self.failUnlessNodeKeysAre, [])
3081         return d
3082
3083     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3084         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3085         d.addBoth(self.shouldFail, error.Error,
3086                   "POST_mkdir_no_replace_queryarg",
3087                   "409 Conflict",
3088                   "There was already a child by that name, and you asked me "
3089                   "to not replace it")
3090         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3091         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3092         return d
3093
3094     def test_POST_mkdir_no_replace_field(self): # return value?
3095         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3096                       replace="false")
3097         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3098                   "409 Conflict",
3099                   "There was already a child by that name, and you asked me "
3100                   "to not replace it")
3101         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3102         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3103         return d
3104
3105     def test_POST_mkdir_whendone_field(self):
3106         d = self.POST(self.public_url + "/foo",
3107                       t="mkdir", name="newdir", when_done="/THERE")
3108         d.addBoth(self.shouldRedirect, "/THERE")
3109         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3110         d.addCallback(self.failUnlessNodeKeysAre, [])
3111         return d
3112
3113     def test_POST_mkdir_whendone_queryarg(self):
3114         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3115                       t="mkdir", name="newdir")
3116         d.addBoth(self.shouldRedirect, "/THERE")
3117         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3118         d.addCallback(self.failUnlessNodeKeysAre, [])
3119         return d
3120
3121     def test_POST_bad_t(self):
3122         d = self.shouldFail2(error.Error, "POST_bad_t",
3123                              "400 Bad Request",
3124                              "POST to a directory with bad t=BOGUS",
3125                              self.POST, self.public_url + "/foo", t="BOGUS")
3126         return d
3127
3128     def test_POST_set_children(self, command_name="set_children"):
3129         contents9, n9, newuri9 = self.makefile(9)
3130         contents10, n10, newuri10 = self.makefile(10)
3131         contents11, n11, newuri11 = self.makefile(11)
3132
3133         reqbody = """{
3134                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3135                                                 "size": 0,
3136                                                 "metadata": {
3137                                                   "ctime": 1002777696.7564139,
3138                                                   "mtime": 1002777696.7564139
3139                                                  }
3140                                                } ],
3141                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3142                                                 "size": 1,
3143                                                 "metadata": {
3144                                                   "ctime": 1002777696.7564139,
3145                                                   "mtime": 1002777696.7564139
3146                                                  }
3147                                                } ],
3148                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3149                                                 "size": 2,
3150                                                 "metadata": {
3151                                                   "ctime": 1002777696.7564139,
3152                                                   "mtime": 1002777696.7564139
3153                                                  }
3154                                                } ]
3155                     }""" % (newuri9, newuri10, newuri11)
3156
3157         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3158
3159         d = client.getPage(url, method="POST", postdata=reqbody)
3160         def _then(res):
3161             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3162             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3163             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3164
3165         d.addCallback(_then)
3166         d.addErrback(self.dump_error)
3167         return d
3168
3169     def test_POST_set_children_with_hyphen(self):
3170         return self.test_POST_set_children(command_name="set-children")
3171
3172     def test_POST_link_uri(self):
3173         contents, n, newuri = self.makefile(8)
3174         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3175         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3176         d.addCallback(lambda res:
3177                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3178                                                       contents))
3179         return d
3180
3181     def test_POST_link_uri_replace(self):
3182         contents, n, newuri = self.makefile(8)
3183         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3184         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3185         d.addCallback(lambda res:
3186                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3187                                                       contents))
3188         return d
3189
3190     def test_POST_link_uri_unknown_bad(self):
3191         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3192         d.addBoth(self.shouldFail, error.Error,
3193                   "POST_link_uri_unknown_bad",
3194                   "400 Bad Request",
3195                   "unknown cap in a write slot")
3196         return d
3197
3198     def test_POST_link_uri_unknown_ro_good(self):
3199         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3200         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3201         return d
3202
3203     def test_POST_link_uri_unknown_imm_good(self):
3204         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3205         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3206         return d
3207
3208     def test_POST_link_uri_no_replace_queryarg(self):
3209         contents, n, newuri = self.makefile(8)
3210         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3211                       name="bar.txt", uri=newuri)
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_link_uri_no_replace_queryarg",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3218         d.addCallback(self.failUnlessIsBarDotTxt)
3219         return d
3220
3221     def test_POST_link_uri_no_replace_field(self):
3222         contents, n, newuri = self.makefile(8)
3223         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3224                       name="bar.txt", uri=newuri)
3225         d.addBoth(self.shouldFail, error.Error,
3226                   "POST_link_uri_no_replace_field",
3227                   "409 Conflict",
3228                   "There was already a child by that name, and you asked me "
3229                   "to not replace it")
3230         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3231         d.addCallback(self.failUnlessIsBarDotTxt)
3232         return d
3233
3234     def test_POST_delete(self, command_name='delete'):
3235         d = self._foo_node.list()
3236         def _check_before(children):
3237             self.failUnless(u"bar.txt" in children)
3238         d.addCallback(_check_before)
3239         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3240         d.addCallback(lambda res: self._foo_node.list())
3241         def _check_after(children):
3242             self.failIf(u"bar.txt" in children)
3243         d.addCallback(_check_after)
3244         return d
3245
3246     def test_POST_unlink(self):
3247         return self.test_POST_delete(command_name='unlink')
3248
3249     def test_POST_rename_file(self):
3250         d = self.POST(self.public_url + "/foo", t="rename",
3251                       from_name="bar.txt", to_name='wibble.txt')
3252         d.addCallback(lambda res:
3253                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3254         d.addCallback(lambda res:
3255                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3257         d.addCallback(self.failUnlessIsBarDotTxt)
3258         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3259         d.addCallback(self.failUnlessIsBarJSON)
3260         return d
3261
3262     def test_POST_rename_file_redundant(self):
3263         d = self.POST(self.public_url + "/foo", t="rename",
3264                       from_name="bar.txt", to_name='bar.txt')
3265         d.addCallback(lambda res:
3266                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3267         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3268         d.addCallback(self.failUnlessIsBarDotTxt)
3269         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3270         d.addCallback(self.failUnlessIsBarJSON)
3271         return d
3272
3273     def test_POST_rename_file_replace(self):
3274         # rename a file and replace a directory with it
3275         d = self.POST(self.public_url + "/foo", t="rename",
3276                       from_name="bar.txt", to_name='empty')
3277         d.addCallback(lambda res:
3278                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3279         d.addCallback(lambda res:
3280                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3281         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3282         d.addCallback(self.failUnlessIsBarDotTxt)
3283         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3284         d.addCallback(self.failUnlessIsBarJSON)
3285         return d
3286
3287     def test_POST_rename_file_no_replace_queryarg(self):
3288         # rename a file and replace a directory with it
3289         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3290                       from_name="bar.txt", to_name='empty')
3291         d.addBoth(self.shouldFail, error.Error,
3292                   "POST_rename_file_no_replace_queryarg",
3293                   "409 Conflict",
3294                   "There was already a child by that name, and you asked me "
3295                   "to not replace it")
3296         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3297         d.addCallback(self.failUnlessIsEmptyJSON)
3298         return d
3299
3300     def test_POST_rename_file_no_replace_field(self):
3301         # rename a file and replace a directory with it
3302         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3303                       from_name="bar.txt", to_name='empty')
3304         d.addBoth(self.shouldFail, error.Error,
3305                   "POST_rename_file_no_replace_field",
3306                   "409 Conflict",
3307                   "There was already a child by that name, and you asked me "
3308                   "to not replace it")
3309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3310         d.addCallback(self.failUnlessIsEmptyJSON)
3311         return d
3312
3313     def failUnlessIsEmptyJSON(self, res):
3314         data = simplejson.loads(res)
3315         self.failUnlessEqual(data[0], "dirnode", data)
3316         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3317
3318     def test_POST_rename_file_slash_fail(self):
3319         d = self.POST(self.public_url + "/foo", t="rename",
3320                       from_name="bar.txt", to_name='kirk/spock.txt')
3321         d.addBoth(self.shouldFail, error.Error,
3322                   "test_POST_rename_file_slash_fail",
3323                   "400 Bad Request",
3324                   "to_name= may not contain a slash",
3325                   )
3326         d.addCallback(lambda res:
3327                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3328         return d
3329
3330     def test_POST_rename_dir(self):
3331         d = self.POST(self.public_url, t="rename",
3332                       from_name="foo", to_name='plunk')
3333         d.addCallback(lambda res:
3334                       self.failIfNodeHasChild(self.public_root, u"foo"))
3335         d.addCallback(lambda res:
3336                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3337         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3338         d.addCallback(self.failUnlessIsFooJSON)
3339         return d
3340
3341     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3342         """ If target is not None then the redirection has to go to target.  If
3343         statuscode is not None then the redirection has to be accomplished with
3344         that HTTP status code."""
3345         if not isinstance(res, failure.Failure):
3346             to_where = (target is None) and "somewhere" or ("to " + target)
3347             self.fail("%s: we were expecting to get redirected %s, not get an"
3348                       " actual page: %s" % (which, to_where, res))
3349         res.trap(error.PageRedirect)
3350         if statuscode is not None:
3351             self.failUnlessReallyEqual(res.value.status, statuscode,
3352                                        "%s: not a redirect" % which)
3353         if target is not None:
3354             # the PageRedirect does not seem to capture the uri= query arg
3355             # properly, so we can't check for it.
3356             realtarget = self.webish_url + target
3357             self.failUnlessReallyEqual(res.value.location, realtarget,
3358                                        "%s: wrong target" % which)
3359         return res.value.location
3360
3361     def test_GET_URI_form(self):
3362         base = "/uri?uri=%s" % self._bar_txt_uri
3363         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3364         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3365         d = self.GET(base)
3366         d.addBoth(self.shouldRedirect, targetbase)
3367         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3368         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3369         d.addCallback(lambda res: self.GET(base+"&t=json"))
3370         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3371         d.addCallback(self.log, "about to get file by uri")
3372         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3373         d.addCallback(self.failUnlessIsBarDotTxt)
3374         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3375         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3376                                            followRedirect=True))
3377         d.addCallback(self.failUnlessIsFooJSON)
3378         d.addCallback(self.log, "got dir by uri")
3379
3380         return d
3381
3382     def test_GET_URI_form_bad(self):
3383         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3384                              "400 Bad Request", "GET /uri requires uri=",
3385                              self.GET, "/uri")
3386         return d
3387
3388     def test_GET_rename_form(self):
3389         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3390                      followRedirect=True)
3391         def _check(res):
3392             self.failUnless('name="when_done" value="."' in res, res)
3393             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3394         d.addCallback(_check)
3395         return d
3396
3397     def log(self, res, msg):
3398         #print "MSG: %s  RES: %s" % (msg, res)
3399         log.msg(msg)
3400         return res
3401
3402     def test_GET_URI_URL(self):
3403         base = "/uri/%s" % self._bar_txt_uri
3404         d = self.GET(base)
3405         d.addCallback(self.failUnlessIsBarDotTxt)
3406         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3407         d.addCallback(self.failUnlessIsBarDotTxt)
3408         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3409         d.addCallback(self.failUnlessIsBarDotTxt)
3410         return d
3411
3412     def test_GET_URI_URL_dir(self):
3413         base = "/uri/%s?t=json" % self._foo_uri
3414         d = self.GET(base)
3415         d.addCallback(self.failUnlessIsFooJSON)
3416         return d
3417
3418     def test_GET_URI_URL_missing(self):
3419         base = "/uri/%s" % self._bad_file_uri
3420         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3421                                  http.GONE, None, "NotEnoughSharesError",
3422                                  self.GET, base)
3423         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3424         # here? we must arrange for a download to fail after target.open()
3425         # has been called, and then inspect the response to see that it is
3426         # shorter than we expected.
3427         return d
3428
3429     def test_PUT_DIRURL_uri(self):
3430         d = self.s.create_dirnode()
3431         def _made_dir(dn):
3432             new_uri = dn.get_uri()
3433             # replace /foo with a new (empty) directory
3434             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3435             d.addCallback(lambda res:
3436                           self.failUnlessReallyEqual(res.strip(), new_uri))
3437             d.addCallback(lambda res:
3438                           self.failUnlessRWChildURIIs(self.public_root,
3439                                                       u"foo",
3440                                                       new_uri))
3441             return d
3442         d.addCallback(_made_dir)
3443         return d
3444
3445     def test_PUT_DIRURL_uri_noreplace(self):
3446         d = self.s.create_dirnode()
3447         def _made_dir(dn):
3448             new_uri = dn.get_uri()
3449             # replace /foo with a new (empty) directory, but ask that
3450             # replace=false, so it should fail
3451             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3452                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3453                                  self.PUT,
3454                                  self.public_url + "/foo?t=uri&replace=false",
3455                                  new_uri)
3456             d.addCallback(lambda res:
3457                           self.failUnlessRWChildURIIs(self.public_root,
3458                                                       u"foo",
3459                                                       self._foo_uri))
3460             return d
3461         d.addCallback(_made_dir)
3462         return d
3463
3464     def test_PUT_DIRURL_bad_t(self):
3465         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3466                              "400 Bad Request", "PUT to a directory",
3467                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3468         d.addCallback(lambda res:
3469                       self.failUnlessRWChildURIIs(self.public_root,
3470                                                   u"foo",
3471                                                   self._foo_uri))
3472         return d
3473
3474     def test_PUT_NEWFILEURL_uri(self):
3475         contents, n, new_uri = self.makefile(8)
3476         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3477         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3478         d.addCallback(lambda res:
3479                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3480                                                       contents))
3481         return d
3482
3483     def test_PUT_NEWFILEURL_mdmf(self):
3484         new_contents = self.NEWFILE_CONTENTS * 300000
3485         d = self.PUT(self.public_url + \
3486                      "/foo/mdmf.txt?format=mdmf",
3487                      new_contents)
3488         d.addCallback(lambda ignored:
3489             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3490         def _got_json(json):
3491             data = simplejson.loads(json)
3492             data = data[1]
3493             self.failUnlessIn("format", data)
3494             self.failUnlessEqual(data["format"], "mdmf")
3495             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3496             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3497         d.addCallback(_got_json)
3498         return d
3499
3500     def test_PUT_NEWFILEURL_sdmf(self):
3501         new_contents = self.NEWFILE_CONTENTS * 300000
3502         d = self.PUT(self.public_url + \
3503                      "/foo/sdmf.txt?format=sdmf",
3504                      new_contents)
3505         d.addCallback(lambda ignored:
3506             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3507         def _got_json(json):
3508             data = simplejson.loads(json)
3509             data = data[1]
3510             self.failUnlessIn("format", data)
3511             self.failUnlessEqual(data["format"], "sdmf")
3512         d.addCallback(_got_json)
3513         return d
3514
3515     def test_PUT_NEWFILEURL_bad_format(self):
3516        new_contents = self.NEWFILE_CONTENTS * 300000
3517        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3518                                    400, "Bad Request", "Unknown format: foo",
3519                                    self.PUT, self.public_url + \
3520                                    "/foo/foo.txt?format=foo",
3521                                    new_contents)
3522
3523     def test_PUT_NEWFILEURL_uri_replace(self):
3524         contents, n, new_uri = self.makefile(8)
3525         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3526         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3527         d.addCallback(lambda res:
3528                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3529                                                       contents))
3530         return d
3531
3532     def test_PUT_NEWFILEURL_uri_no_replace(self):
3533         contents, n, new_uri = self.makefile(8)
3534         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3535         d.addBoth(self.shouldFail, error.Error,
3536                   "PUT_NEWFILEURL_uri_no_replace",
3537                   "409 Conflict",
3538                   "There was already a child by that name, and you asked me "
3539                   "to not replace it")
3540         return d
3541
3542     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3543         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3544         d.addBoth(self.shouldFail, error.Error,
3545                   "POST_put_uri_unknown_bad",
3546                   "400 Bad Request",
3547                   "unknown cap in a write slot")
3548         return d
3549
3550     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3551         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3552         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3553                       u"put-future-ro.txt")
3554         return d
3555
3556     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3557         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3558         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3559                       u"put-future-imm.txt")
3560         return d
3561
3562     def test_PUT_NEWFILE_URI(self):
3563         file_contents = "New file contents here\n"
3564         d = self.PUT("/uri", file_contents)
3565         def _check(uri):
3566             assert isinstance(uri, str), uri
3567             self.failUnless(uri in FakeCHKFileNode.all_contents)
3568             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3569                                        file_contents)
3570             return self.GET("/uri/%s" % uri)
3571         d.addCallback(_check)
3572         def _check2(res):
3573             self.failUnlessReallyEqual(res, file_contents)
3574         d.addCallback(_check2)
3575         return d
3576
3577     def test_PUT_NEWFILE_URI_not_mutable(self):
3578         file_contents = "New file contents here\n"
3579         d = self.PUT("/uri?mutable=false", file_contents)
3580         def _check(uri):
3581             assert isinstance(uri, str), uri
3582             self.failUnless(uri in FakeCHKFileNode.all_contents)
3583             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3584                                        file_contents)
3585             return self.GET("/uri/%s" % uri)
3586         d.addCallback(_check)
3587         def _check2(res):
3588             self.failUnlessReallyEqual(res, file_contents)
3589         d.addCallback(_check2)
3590         return d
3591
3592     def test_PUT_NEWFILE_URI_only_PUT(self):
3593         d = self.PUT("/uri?t=bogus", "")
3594         d.addBoth(self.shouldFail, error.Error,
3595                   "PUT_NEWFILE_URI_only_PUT",
3596                   "400 Bad Request",
3597                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3598         return d
3599
3600     def test_PUT_NEWFILE_URI_mutable(self):
3601         file_contents = "New file contents here\n"
3602         d = self.PUT("/uri?mutable=true", file_contents)
3603         def _check1(filecap):
3604             filecap = filecap.strip()
3605             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3606             self.filecap = filecap
3607             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3608             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3609             n = self.s.create_node_from_uri(filecap)
3610             return n.download_best_version()
3611         d.addCallback(_check1)
3612         def _check2(data):
3613             self.failUnlessReallyEqual(data, file_contents)
3614             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3615         d.addCallback(_check2)
3616         def _check3(res):
3617             self.failUnlessReallyEqual(res, file_contents)
3618         d.addCallback(_check3)
3619         return d
3620
3621     def test_PUT_mkdir(self):
3622         d = self.PUT("/uri?t=mkdir", "")
3623         def _check(uri):
3624             n = self.s.create_node_from_uri(uri.strip())
3625             d2 = self.failUnlessNodeKeysAre(n, [])
3626             d2.addCallback(lambda res:
3627                            self.GET("/uri/%s?t=json" % uri))
3628             return d2
3629         d.addCallback(_check)
3630         d.addCallback(self.failUnlessIsEmptyJSON)
3631         return d
3632
3633     def test_PUT_mkdir_mdmf(self):
3634         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3635         def _got(res):
3636             u = uri.from_string(res)
3637             # Check that this is an MDMF writecap
3638             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3639         d.addCallback(_got)
3640         return d
3641
3642     def test_PUT_mkdir_sdmf(self):
3643         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3644         def _got(res):
3645             u = uri.from_string(res)
3646             self.failUnlessIsInstance(u, uri.DirectoryURI)
3647         d.addCallback(_got)
3648         return d
3649
3650     def test_PUT_mkdir_bad_format(self):
3651         return self.shouldHTTPError("PUT_mkdir_bad_format",
3652                                     400, "Bad Request", "Unknown format: foo",
3653                                     self.PUT, "/uri?t=mkdir&format=foo",
3654                                     "")
3655
3656     def test_POST_check(self):
3657         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3658         def _done(res):
3659             # this returns a string form of the results, which are probably
3660             # None since we're using fake filenodes.
3661             # TODO: verify that the check actually happened, by changing
3662             # FakeCHKFileNode to count how many times .check() has been
3663             # called.
3664             pass
3665         d.addCallback(_done)
3666         return d
3667
3668
3669     def test_PUT_update_at_offset(self):
3670         file_contents = "test file" * 100000 # about 900 KiB
3671         d = self.PUT("/uri?mutable=true", file_contents)
3672         def _then(filecap):
3673             self.filecap = filecap
3674             new_data = file_contents[:100]
3675             new = "replaced and so on"
3676             new_data += new
3677             new_data += file_contents[len(new_data):]
3678             assert len(new_data) == len(file_contents)
3679             self.new_data = new_data
3680         d.addCallback(_then)
3681         d.addCallback(lambda ignored:
3682             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3683                      "replaced and so on"))
3684         def _get_data(filecap):
3685             n = self.s.create_node_from_uri(filecap)
3686             return n.download_best_version()
3687         d.addCallback(_get_data)
3688         d.addCallback(lambda results:
3689             self.failUnlessEqual(results, self.new_data))
3690         # Now try appending things to the file
3691         d.addCallback(lambda ignored:
3692             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3693                      "puppies" * 100))
3694         d.addCallback(_get_data)
3695         d.addCallback(lambda results:
3696             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3697         # and try replacing the beginning of the file
3698         d.addCallback(lambda ignored:
3699             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3700         d.addCallback(_get_data)
3701         d.addCallback(lambda results:
3702             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3703         return d
3704
3705     def test_PUT_update_at_invalid_offset(self):
3706         file_contents = "test file" * 100000 # about 900 KiB
3707         d = self.PUT("/uri?mutable=true", file_contents)
3708         def _then(filecap):
3709             self.filecap = filecap
3710         d.addCallback(_then)
3711         # Negative offsets should cause an error.
3712         d.addCallback(lambda ignored:
3713             self.shouldHTTPError("PUT_update_at_invalid_offset",
3714                                  400, "Bad Request",
3715                                  "Invalid offset",
3716                                  self.PUT,
3717                                  "/uri/%s?offset=-1" % self.filecap,
3718                                  "foo"))
3719         return d
3720
3721     def test_PUT_update_at_offset_immutable(self):
3722         file_contents = "Test file" * 100000
3723         d = self.PUT("/uri", file_contents)
3724         def _then(filecap):
3725             self.filecap = filecap
3726         d.addCallback(_then)
3727         d.addCallback(lambda ignored:
3728             self.shouldHTTPError("PUT_update_at_offset_immutable",
3729                                  400, "Bad Request",
3730                                  "immutable",
3731                                  self.PUT,
3732                                  "/uri/%s?offset=50" % self.filecap,
3733                                  "foo"))
3734         return d
3735
3736
3737     def test_bad_method(self):
3738         url = self.webish_url + self.public_url + "/foo/bar.txt"
3739         d = self.shouldHTTPError("bad_method",
3740                                  501, "Not Implemented",
3741                                  "I don't know how to treat a BOGUS request.",
3742                                  client.getPage, url, method="BOGUS")
3743         return d
3744
3745     def test_short_url(self):
3746         url = self.webish_url + "/uri"
3747         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3748                                  "I don't know how to treat a DELETE request.",
3749                                  client.getPage, url, method="DELETE")
3750         return d
3751
3752     def test_ophandle_bad(self):
3753         url = self.webish_url + "/operations/bogus?t=status"
3754         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3755                                  "unknown/expired handle 'bogus'",
3756                                  client.getPage, url)
3757         return d
3758
3759     def test_ophandle_cancel(self):
3760         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3761                       followRedirect=True)
3762         d.addCallback(lambda ignored:
3763                       self.GET("/operations/128?t=status&output=JSON"))
3764         def _check1(res):
3765             data = simplejson.loads(res)
3766             self.failUnless("finished" in data, res)
3767             monitor = self.ws.root.child_operations.handles["128"][0]
3768             d = self.POST("/operations/128?t=cancel&output=JSON")
3769             def _check2(res):
3770                 data = simplejson.loads(res)
3771                 self.failUnless("finished" in data, res)
3772                 # t=cancel causes the handle to be forgotten
3773                 self.failUnless(monitor.is_cancelled())
3774             d.addCallback(_check2)
3775             return d
3776         d.addCallback(_check1)
3777         d.addCallback(lambda ignored:
3778                       self.shouldHTTPError("ophandle_cancel",
3779                                            404, "404 Not Found",
3780                                            "unknown/expired handle '128'",
3781                                            self.GET,
3782                                            "/operations/128?t=status&output=JSON"))
3783         return d
3784
3785     def test_ophandle_retainfor(self):
3786         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3787                       followRedirect=True)
3788         d.addCallback(lambda ignored:
3789                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3790         def _check1(res):
3791             data = simplejson.loads(res)
3792             self.failUnless("finished" in data, res)
3793         d.addCallback(_check1)
3794         # the retain-for=0 will cause the handle to be expired very soon
3795         d.addCallback(lambda ign:
3796             self.clock.advance(2.0))
3797         d.addCallback(lambda ignored:
3798                       self.shouldHTTPError("ophandle_retainfor",
3799                                            404, "404 Not Found",
3800                                            "unknown/expired handle '129'",
3801                                            self.GET,
3802                                            "/operations/129?t=status&output=JSON"))
3803         return d
3804
3805     def test_ophandle_release_after_complete(self):
3806         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3807                       followRedirect=True)
3808         d.addCallback(self.wait_for_operation, "130")
3809         d.addCallback(lambda ignored:
3810                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3811         # the release-after-complete=true will cause the handle to be expired
3812         d.addCallback(lambda ignored:
3813                       self.shouldHTTPError("ophandle_release_after_complete",
3814                                            404, "404 Not Found",
3815                                            "unknown/expired handle '130'",
3816                                            self.GET,
3817                                            "/operations/130?t=status&output=JSON"))
3818         return d
3819
3820     def test_uncollected_ophandle_expiration(self):
3821         # uncollected ophandles should expire after 4 days
3822         def _make_uncollected_ophandle(ophandle):
3823             d = self.POST(self.public_url +
3824                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3825                           followRedirect=False)
3826             # When we start the operation, the webapi server will want
3827             # to redirect us to the page for the ophandle, so we get
3828             # confirmation that the operation has started. If the
3829             # manifest operation has finished by the time we get there,
3830             # following that redirect (by setting followRedirect=True
3831             # above) has the side effect of collecting the ophandle that
3832             # we've just created, which means that we can't use the
3833             # ophandle to test the uncollected timeout anymore. So,
3834             # instead, catch the 302 here and don't follow it.
3835             d.addBoth(self.should302, "uncollected_ophandle_creation")
3836             return d
3837         # Create an ophandle, don't collect it, then advance the clock by
3838         # 4 days - 1 second and make sure that the ophandle is still there.
3839         d = _make_uncollected_ophandle(131)
3840         d.addCallback(lambda ign:
3841             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3842         d.addCallback(lambda ign:
3843             self.GET("/operations/131?t=status&output=JSON"))
3844         def _check1(res):
3845             data = simplejson.loads(res)
3846             self.failUnless("finished" in data, res)
3847         d.addCallback(_check1)
3848         # Create an ophandle, don't collect it, then try to collect it
3849         # after 4 days. It should be gone.
3850         d.addCallback(lambda ign:
3851             _make_uncollected_ophandle(132))
3852         d.addCallback(lambda ign:
3853             self.clock.advance(96*60*60))
3854         d.addCallback(lambda ign:
3855             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3856                                  404, "404 Not Found",
3857                                  "unknown/expired handle '132'",
3858                                  self.GET,
3859                                  "/operations/132?t=status&output=JSON"))
3860         return d
3861
3862     def test_collected_ophandle_expiration(self):
3863         # collected ophandles should expire after 1 day
3864         def _make_collected_ophandle(ophandle):
3865             d = self.POST(self.public_url +
3866                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3867                           followRedirect=True)
3868             # By following the initial redirect, we collect the ophandle
3869             # we've just created.
3870             return d
3871         # Create a collected ophandle, then collect it after 23 hours
3872         # and 59 seconds to make sure that it is still there.
3873         d = _make_collected_ophandle(133)
3874         d.addCallback(lambda ign:
3875             self.clock.advance((24*60*60) - 1))
3876         d.addCallback(lambda ign:
3877             self.GET("/operations/133?t=status&output=JSON"))
3878         def _check1(res):
3879             data = simplejson.loads(res)
3880             self.failUnless("finished" in data, res)
3881         d.addCallback(_check1)
3882         # Create another uncollected ophandle, then try to collect it
3883         # after 24 hours to make sure that it is gone.
3884         d.addCallback(lambda ign:
3885             _make_collected_ophandle(134))
3886         d.addCallback(lambda ign:
3887             self.clock.advance(24*60*60))
3888         d.addCallback(lambda ign:
3889             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3890                                  404, "404 Not Found",
3891                                  "unknown/expired handle '134'",
3892                                  self.GET,
3893                                  "/operations/134?t=status&output=JSON"))
3894         return d
3895
3896     def test_incident(self):
3897         d = self.POST("/report_incident", details="eek")
3898         def _done(res):
3899             self.failUnless("Thank you for your report!" in res, res)
3900         d.addCallback(_done)
3901         return d
3902
3903     def test_static(self):
3904         webdir = os.path.join(self.staticdir, "subdir")
3905         fileutil.make_dirs(webdir)
3906         f = open(os.path.join(webdir, "hello.txt"), "wb")
3907         f.write("hello")
3908         f.close()
3909
3910         d = self.GET("/static/subdir/hello.txt")
3911         def _check(res):
3912             self.failUnlessReallyEqual(res, "hello")
3913         d.addCallback(_check)
3914         return d
3915
3916
3917 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3918     def test_load_file(self):
3919         # This will raise an exception unless a well-formed XML file is found under that name.
3920         common.getxmlfile('directory.xhtml').load()
3921
3922     def test_parse_replace_arg(self):
3923         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3924         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3925         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3926                                    "only-files")
3927         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3928                         common.parse_replace_arg, "only_fles")
3929
3930     def test_abbreviate_time(self):
3931         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3932         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3933         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3934         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3935         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3936         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3937
3938     def test_compute_rate(self):
3939         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3940         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3941         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3942         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3943         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3944         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3945         self.shouldFail(AssertionError, "test_compute_rate", "",
3946                         common.compute_rate, -100, 10)
3947         self.shouldFail(AssertionError, "test_compute_rate", "",
3948                         common.compute_rate, 100, -10)
3949
3950         # Sanity check
3951         rate = common.compute_rate(10*1000*1000, 1)
3952         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3953
3954     def test_abbreviate_rate(self):
3955         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3956         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3957         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3958         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3959
3960     def test_abbreviate_size(self):
3961         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3962         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3963         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3964         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3965         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3966
3967     def test_plural(self):
3968         def convert(s):
3969             return "%d second%s" % (s, status.plural(s))
3970         self.failUnlessReallyEqual(convert(0), "0 seconds")
3971         self.failUnlessReallyEqual(convert(1), "1 second")
3972         self.failUnlessReallyEqual(convert(2), "2 seconds")
3973         def convert2(s):
3974             return "has share%s: %s" % (status.plural(s), ",".join(s))
3975         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3976         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3977         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3978
3979
3980 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3981
3982     def CHECK(self, ign, which, args, clientnum=0):
3983         fileurl = self.fileurls[which]
3984         url = fileurl + "?" + args
3985         return self.GET(url, method="POST", clientnum=clientnum)
3986
3987     def test_filecheck(self):
3988         self.basedir = "web/Grid/filecheck"
3989         self.set_up_grid()
3990         c0 = self.g.clients[0]
3991         self.uris = {}
3992         DATA = "data" * 100
3993         d = c0.upload(upload.Data(DATA, convergence=""))
3994         def _stash_uri(ur, which):
3995             self.uris[which] = ur.uri
3996         d.addCallback(_stash_uri, "good")
3997         d.addCallback(lambda ign:
3998                       c0.upload(upload.Data(DATA+"1", convergence="")))
3999         d.addCallback(_stash_uri, "sick")
4000         d.addCallback(lambda ign:
4001                       c0.upload(upload.Data(DATA+"2", convergence="")))
4002         d.addCallback(_stash_uri, "dead")
4003         def _stash_mutable_uri(n, which):
4004             self.uris[which] = n.get_uri()
4005             assert isinstance(self.uris[which], str)
4006         d.addCallback(lambda ign:
4007             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4008         d.addCallback(_stash_mutable_uri, "corrupt")
4009         d.addCallback(lambda ign:
4010                       c0.upload(upload.Data("literal", convergence="")))
4011         d.addCallback(_stash_uri, "small")
4012         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4013         d.addCallback(_stash_mutable_uri, "smalldir")
4014
4015         def _compute_fileurls(ignored):
4016             self.fileurls = {}
4017             for which in self.uris:
4018                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4019         d.addCallback(_compute_fileurls)
4020
4021         def _clobber_shares(ignored):
4022             good_shares = self.find_uri_shares(self.uris["good"])
4023             self.failUnlessReallyEqual(len(good_shares), 10)
4024             sick_shares = self.find_uri_shares(self.uris["sick"])
4025             os.unlink(sick_shares[0][2])
4026             dead_shares = self.find_uri_shares(self.uris["dead"])
4027             for i in range(1, 10):
4028                 os.unlink(dead_shares[i][2])
4029             c_shares = self.find_uri_shares(self.uris["corrupt"])
4030             cso = CorruptShareOptions()
4031             cso.stdout = StringIO()
4032             cso.parseOptions([c_shares[0][2]])
4033             corrupt_share(cso)
4034         d.addCallback(_clobber_shares)
4035
4036         d.addCallback(self.CHECK, "good", "t=check")
4037         def _got_html_good(res):
4038             self.failUnless("Healthy" in res, res)
4039             self.failIf("Not Healthy" in res, res)
4040         d.addCallback(_got_html_good)
4041         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4042         def _got_html_good_return_to(res):
4043             self.failUnless("Healthy" in res, res)
4044             self.failIf("Not Healthy" in res, res)
4045             self.failUnless('<a href="somewhere">Return to file'
4046                             in res, res)
4047         d.addCallback(_got_html_good_return_to)
4048         d.addCallback(self.CHECK, "good", "t=check&output=json")
4049         def _got_json_good(res):
4050             r = simplejson.loads(res)
4051             self.failUnlessEqual(r["summary"], "Healthy")
4052             self.failUnless(r["results"]["healthy"])
4053             self.failIf(r["results"]["needs-rebalancing"])
4054             self.failUnless(r["results"]["recoverable"])
4055         d.addCallback(_got_json_good)
4056
4057         d.addCallback(self.CHECK, "small", "t=check")
4058         def _got_html_small(res):
4059             self.failUnless("Literal files are always healthy" in res, res)
4060             self.failIf("Not Healthy" in res, res)
4061         d.addCallback(_got_html_small)
4062         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4063         def _got_html_small_return_to(res):
4064             self.failUnless("Literal files are always healthy" in res, res)
4065             self.failIf("Not Healthy" in res, res)
4066             self.failUnless('<a href="somewhere">Return to file'
4067                             in res, res)
4068         d.addCallback(_got_html_small_return_to)
4069         d.addCallback(self.CHECK, "small", "t=check&output=json")
4070         def _got_json_small(res):
4071             r = simplejson.loads(res)
4072             self.failUnlessEqual(r["storage-index"], "")
4073             self.failUnless(r["results"]["healthy"])
4074         d.addCallback(_got_json_small)
4075
4076         d.addCallback(self.CHECK, "smalldir", "t=check")
4077         def _got_html_smalldir(res):
4078             self.failUnless("Literal files are always healthy" in res, res)
4079             self.failIf("Not Healthy" in res, res)
4080         d.addCallback(_got_html_smalldir)
4081         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4082         def _got_json_smalldir(res):
4083             r = simplejson.loads(res)
4084             self.failUnlessEqual(r["storage-index"], "")
4085             self.failUnless(r["results"]["healthy"])
4086         d.addCallback(_got_json_smalldir)
4087
4088         d.addCallback(self.CHECK, "sick", "t=check")
4089         def _got_html_sick(res):
4090             self.failUnless("Not Healthy" in res, res)
4091         d.addCallback(_got_html_sick)
4092         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4093         def _got_json_sick(res):
4094             r = simplejson.loads(res)
4095             self.failUnlessEqual(r["summary"],
4096                                  "Not Healthy: 9 shares (enc 3-of-10)")
4097             self.failIf(r["results"]["healthy"])
4098             self.failIf(r["results"]["needs-rebalancing"])
4099             self.failUnless(r["results"]["recoverable"])
4100         d.addCallback(_got_json_sick)
4101
4102         d.addCallback(self.CHECK, "dead", "t=check")
4103         def _got_html_dead(res):
4104             self.failUnless("Not Healthy" in res, res)
4105         d.addCallback(_got_html_dead)
4106         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4107         def _got_json_dead(res):
4108             r = simplejson.loads(res)
4109             self.failUnlessEqual(r["summary"],
4110                                  "Not Healthy: 1 shares (enc 3-of-10)")
4111             self.failIf(r["results"]["healthy"])
4112             self.failIf(r["results"]["needs-rebalancing"])
4113             self.failIf(r["results"]["recoverable"])
4114         d.addCallback(_got_json_dead)
4115
4116         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4117         def _got_html_corrupt(res):
4118             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4119         d.addCallback(_got_html_corrupt)
4120         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4121         def _got_json_corrupt(res):
4122             r = simplejson.loads(res)
4123             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4124                             r["summary"])
4125             self.failIf(r["results"]["healthy"])
4126             self.failUnless(r["results"]["recoverable"])
4127             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4128             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4129         d.addCallback(_got_json_corrupt)
4130
4131         d.addErrback(self.explain_web_error)
4132         return d
4133
4134     def test_repair_html(self):
4135         self.basedir = "web/Grid/repair_html"
4136         self.set_up_grid()
4137         c0 = self.g.clients[0]
4138         self.uris = {}
4139         DATA = "data" * 100
4140         d = c0.upload(upload.Data(DATA, convergence=""))
4141         def _stash_uri(ur, which):
4142             self.uris[which] = ur.uri
4143         d.addCallback(_stash_uri, "good")
4144         d.addCallback(lambda ign:
4145                       c0.upload(upload.Data(DATA+"1", convergence="")))
4146         d.addCallback(_stash_uri, "sick")
4147         d.addCallback(lambda ign:
4148                       c0.upload(upload.Data(DATA+"2", convergence="")))
4149         d.addCallback(_stash_uri, "dead")
4150         def _stash_mutable_uri(n, which):
4151             self.uris[which] = n.get_uri()
4152             assert isinstance(self.uris[which], str)
4153         d.addCallback(lambda ign:
4154             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4155         d.addCallback(_stash_mutable_uri, "corrupt")
4156
4157         def _compute_fileurls(ignored):
4158             self.fileurls = {}
4159             for which in self.uris:
4160                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4161         d.addCallback(_compute_fileurls)
4162
4163         def _clobber_shares(ignored):
4164             good_shares = self.find_uri_shares(self.uris["good"])
4165             self.failUnlessReallyEqual(len(good_shares), 10)
4166             sick_shares = self.find_uri_shares(self.uris["sick"])
4167             os.unlink(sick_shares[0][2])
4168             dead_shares = self.find_uri_shares(self.uris["dead"])
4169             for i in range(1, 10):
4170                 os.unlink(dead_shares[i][2])
4171             c_shares = self.find_uri_shares(self.uris["corrupt"])
4172             cso = CorruptShareOptions()
4173             cso.stdout = StringIO()
4174             cso.parseOptions([c_shares[0][2]])
4175             corrupt_share(cso)
4176         d.addCallback(_clobber_shares)
4177
4178         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4179         def _got_html_good(res):
4180             self.failUnless("Healthy" in res, res)
4181             self.failIf("Not Healthy" in res, res)
4182             self.failUnless("No repair necessary" in res, res)
4183         d.addCallback(_got_html_good)
4184
4185         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4186         def _got_html_sick(res):
4187             self.failUnless("Healthy : healthy" in res, res)
4188             self.failIf("Not Healthy" in res, res)
4189             self.failUnless("Repair successful" in res, res)
4190         d.addCallback(_got_html_sick)
4191
4192         # repair of a dead file will fail, of course, but it isn't yet
4193         # clear how this should be reported. Right now it shows up as
4194         # a "410 Gone".
4195         #
4196         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4197         #def _got_html_dead(res):
4198         #    print res
4199         #    self.failUnless("Healthy : healthy" in res, res)
4200         #    self.failIf("Not Healthy" in res, res)
4201         #    self.failUnless("No repair necessary" in res, res)
4202         #d.addCallback(_got_html_dead)
4203
4204         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4205         def _got_html_corrupt(res):
4206             self.failUnless("Healthy : Healthy" in res, res)
4207             self.failIf("Not Healthy" in res, res)
4208             self.failUnless("Repair successful" in res, res)
4209         d.addCallback(_got_html_corrupt)
4210
4211         d.addErrback(self.explain_web_error)
4212         return d
4213
4214     def test_repair_json(self):
4215         self.basedir = "web/Grid/repair_json"
4216         self.set_up_grid()
4217         c0 = self.g.clients[0]
4218         self.uris = {}
4219         DATA = "data" * 100
4220         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4221         def _stash_uri(ur, which):
4222             self.uris[which] = ur.uri
4223         d.addCallback(_stash_uri, "sick")
4224
4225         def _compute_fileurls(ignored):
4226             self.fileurls = {}
4227             for which in self.uris:
4228                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4229         d.addCallback(_compute_fileurls)
4230
4231         def _clobber_shares(ignored):
4232             sick_shares = self.find_uri_shares(self.uris["sick"])
4233             os.unlink(sick_shares[0][2])
4234         d.addCallback(_clobber_shares)
4235
4236         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4237         def _got_json_sick(res):
4238             r = simplejson.loads(res)
4239             self.failUnlessReallyEqual(r["repair-attempted"], True)
4240             self.failUnlessReallyEqual(r["repair-successful"], True)
4241             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4242                                  "Not Healthy: 9 shares (enc 3-of-10)")
4243             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4244             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4245             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4246         d.addCallback(_got_json_sick)
4247
4248         d.addErrback(self.explain_web_error)
4249         return d
4250
4251     def test_unknown(self, immutable=False):
4252         self.basedir = "web/Grid/unknown"
4253         if immutable:
4254             self.basedir = "web/Grid/unknown-immutable"
4255
4256         self.set_up_grid()
4257         c0 = self.g.clients[0]
4258         self.uris = {}
4259         self.fileurls = {}
4260
4261         # the future cap format may contain slashes, which must be tolerated
4262         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4263                                                            safe="")
4264
4265         if immutable:
4266             name = u"future-imm"
4267             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4268             d = c0.create_immutable_dirnode({name: (future_node, {})})
4269         else:
4270             name = u"future"
4271             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4272             d = c0.create_dirnode()
4273
4274         def _stash_root_and_create_file(n):
4275             self.rootnode = n
4276             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4277             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4278             if not immutable:
4279                 return self.rootnode.set_node(name, future_node)
4280         d.addCallback(_stash_root_and_create_file)
4281
4282         # make sure directory listing tolerates unknown nodes
4283         d.addCallback(lambda ign: self.GET(self.rooturl))
4284         def _check_directory_html(res, expected_type_suffix):
4285             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4286                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4287                                  re.DOTALL)
4288             self.failUnless(re.search(pattern, res), res)
4289             # find the More Info link for name, should be relative
4290             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4291             info_url = mo.group(1)
4292             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4293         if immutable:
4294             d.addCallback(_check_directory_html, "-IMM")
4295         else:
4296             d.addCallback(_check_directory_html, "")
4297
4298         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4299         def _check_directory_json(res, expect_rw_uri):
4300             data = simplejson.loads(res)
4301             self.failUnlessEqual(data[0], "dirnode")
4302             f = data[1]["children"][name]
4303             self.failUnlessEqual(f[0], "unknown")
4304             if expect_rw_uri:
4305                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4306             else:
4307                 self.failIfIn("rw_uri", f[1])
4308             if immutable:
4309                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4310             else:
4311                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4312             self.failUnless("metadata" in f[1])
4313         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4314
4315         def _check_info(res, expect_rw_uri, expect_ro_uri):
4316             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4317             if expect_rw_uri:
4318                 self.failUnlessIn(unknown_rwcap, res)
4319             if expect_ro_uri:
4320                 if immutable:
4321                     self.failUnlessIn(unknown_immcap, res)
4322                 else:
4323                     self.failUnlessIn(unknown_rocap, res)
4324             else:
4325                 self.failIfIn(unknown_rocap, res)
4326             self.failIfIn("Raw data as", res)
4327             self.failIfIn("Directory writecap", res)
4328             self.failIfIn("Checker Operations", res)
4329             self.failIfIn("Mutable File Operations", res)
4330             self.failIfIn("Directory Operations", res)
4331
4332         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4333         # why they fail. Possibly related to ticket #922.
4334
4335         d.addCallback(lambda ign: self.GET(expected_info_url))
4336         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4337         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4338         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4339
4340         def _check_json(res, expect_rw_uri):
4341             data = simplejson.loads(res)
4342             self.failUnlessEqual(data[0], "unknown")
4343             if expect_rw_uri:
4344                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4345             else:
4346                 self.failIfIn("rw_uri", data[1])
4347
4348             if immutable:
4349                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4350                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4351             elif expect_rw_uri:
4352                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4353                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4354             else:
4355                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4356                 self.failIf("mutable" in data[1], data[1])
4357
4358             # TODO: check metadata contents
4359             self.failUnless("metadata" in data[1])
4360
4361         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4362         d.addCallback(_check_json, expect_rw_uri=not immutable)
4363
4364         # and make sure that a read-only version of the directory can be
4365         # rendered too. This version will not have unknown_rwcap, whether
4366         # or not future_node was immutable.
4367         d.addCallback(lambda ign: self.GET(self.rourl))
4368         if immutable:
4369             d.addCallback(_check_directory_html, "-IMM")
4370         else:
4371             d.addCallback(_check_directory_html, "-RO")
4372
4373         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4374         d.addCallback(_check_directory_json, expect_rw_uri=False)
4375
4376         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4377         d.addCallback(_check_json, expect_rw_uri=False)
4378
4379         # TODO: check that getting t=info from the Info link in the ro directory
4380         # works, and does not include the writecap URI.
4381         return d
4382
4383     def test_immutable_unknown(self):
4384         return self.test_unknown(immutable=True)
4385
4386     def test_mutant_dirnodes_are_omitted(self):
4387         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4388
4389         self.set_up_grid()
4390         c = self.g.clients[0]
4391         nm = c.nodemaker
4392         self.uris = {}
4393         self.fileurls = {}
4394
4395         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4396         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4397         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4398
4399         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4400         # test the dirnode and web layers separately.
4401
4402         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4403         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4404         # When the directory is read, the mutants should be silently disposed of, leaving
4405         # their lonely sibling.
4406         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4407         # because immutable directories don't have a writecap and therefore that field
4408         # isn't (and can't be) decrypted.
4409         # TODO: The field still exists in the netstring. Technically we should check what
4410         # happens if something is put there (_unpack_contents should raise ValueError),
4411         # but that can wait.
4412
4413         lonely_child = nm.create_from_cap(lonely_uri)
4414         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4415         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4416
4417         def _by_hook_or_by_crook():
4418             return True
4419         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4420             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4421
4422         mutant_write_in_ro_child.get_write_uri    = lambda: None
4423         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4424
4425         kids = {u"lonely":      (lonely_child, {}),
4426                 u"ro":          (mutant_ro_child, {}),
4427                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4428                 }
4429         d = c.create_immutable_dirnode(kids)
4430
4431         def _created(dn):
4432             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4433             self.failIf(dn.is_mutable())
4434             self.failUnless(dn.is_readonly())
4435             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4436             self.failIf(hasattr(dn._node, 'get_writekey'))
4437             rep = str(dn)
4438             self.failUnless("RO-IMM" in rep)
4439             cap = dn.get_cap()
4440             self.failUnlessIn("CHK", cap.to_string())
4441             self.cap = cap
4442             self.rootnode = dn
4443             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4444             return download_to_data(dn._node)
4445         d.addCallback(_created)
4446
4447         def _check_data(data):
4448             # Decode the netstring representation of the directory to check that all children
4449             # are present. This is a bit of an abstraction violation, but there's not really
4450             # any other way to do it given that the real DirectoryNode._unpack_contents would
4451             # strip the mutant children out (which is what we're trying to test, later).
4452             position = 0
4453             numkids = 0
4454             while position < len(data):
4455                 entries, position = split_netstring(data, 1, position)
4456                 entry = entries[0]
4457                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4458                 name = name_utf8.decode("utf-8")
4459                 self.failUnless(rwcapdata == "")
4460                 self.failUnless(name in kids)
4461                 (expected_child, ign) = kids[name]
4462                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4463                 numkids += 1
4464
4465             self.failUnlessReallyEqual(numkids, 3)
4466             return self.rootnode.list()
4467         d.addCallback(_check_data)
4468
4469         # Now when we use the real directory listing code, the mutants should be absent.
4470         def _check_kids(children):
4471             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4472             lonely_node, lonely_metadata = children[u"lonely"]
4473
4474             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4475             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4476         d.addCallback(_check_kids)
4477
4478         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4479         d.addCallback(lambda n: n.list())
4480         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4481
4482         # Make sure the lonely child can be listed in HTML...
4483         d.addCallback(lambda ign: self.GET(self.rooturl))
4484         def _check_html(res):
4485             self.failIfIn("URI:SSK", res)
4486             get_lonely = "".join([r'<td>FILE</td>',
4487                                   r'\s+<td>',
4488                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4489                                   r'</td>',
4490                                   r'\s+<td align="right">%d</td>' % len("one"),
4491                                   ])
4492             self.failUnless(re.search(get_lonely, res), res)
4493
4494             # find the More Info link for name, should be relative
4495             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4496             info_url = mo.group(1)
4497             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4498         d.addCallback(_check_html)
4499
4500         # ... and in JSON.
4501         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4502         def _check_json(res):
4503             data = simplejson.loads(res)
4504             self.failUnlessEqual(data[0], "dirnode")
4505             listed_children = data[1]["children"]
4506             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4507             ll_type, ll_data = listed_children[u"lonely"]
4508             self.failUnlessEqual(ll_type, "filenode")
4509             self.failIf("rw_uri" in ll_data)
4510             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4511         d.addCallback(_check_json)
4512         return d
4513
4514     def test_deep_check(self):
4515         self.basedir = "web/Grid/deep_check"
4516         self.set_up_grid()
4517         c0 = self.g.clients[0]
4518         self.uris = {}
4519         self.fileurls = {}
4520         DATA = "data" * 100
4521         d = c0.create_dirnode()
4522         def _stash_root_and_create_file(n):
4523             self.rootnode = n
4524             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4525             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4526         d.addCallback(_stash_root_and_create_file)
4527         def _stash_uri(fn, which):
4528             self.uris[which] = fn.get_uri()
4529             return fn
4530         d.addCallback(_stash_uri, "good")
4531         d.addCallback(lambda ign:
4532                       self.rootnode.add_file(u"small",
4533                                              upload.Data("literal",
4534                                                         convergence="")))
4535         d.addCallback(_stash_uri, "small")
4536         d.addCallback(lambda ign:
4537                       self.rootnode.add_file(u"sick",
4538                                              upload.Data(DATA+"1",
4539                                                         convergence="")))
4540         d.addCallback(_stash_uri, "sick")
4541
4542         # this tests that deep-check and stream-manifest will ignore
4543         # UnknownNode instances. Hopefully this will also cover deep-stats.
4544         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4545         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4546
4547         def _clobber_shares(ignored):
4548             self.delete_shares_numbered(self.uris["sick"], [0,1])
4549         d.addCallback(_clobber_shares)
4550
4551         # root
4552         # root/good
4553         # root/small
4554         # root/sick
4555         # root/future
4556
4557         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4558         def _done(res):
4559             try:
4560                 units = [simplejson.loads(line)
4561                          for line in res.splitlines()
4562                          if line]
4563             except ValueError:
4564                 print "response is:", res
4565                 print "undecodeable line was '%s'" % line
4566                 raise
4567             self.failUnlessReallyEqual(len(units), 5+1)
4568             # should be parent-first
4569             u0 = units[0]
4570             self.failUnlessEqual(u0["path"], [])
4571             self.failUnlessEqual(u0["type"], "directory")
4572             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4573             u0cr = u0["check-results"]
4574             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4575
4576             ugood = [u for u in units
4577                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4578             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4579             ugoodcr = ugood["check-results"]
4580             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4581
4582             stats = units[-1]
4583             self.failUnlessEqual(stats["type"], "stats")
4584             s = stats["stats"]
4585             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4586             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4587             self.failUnlessReallyEqual(s["count-directories"], 1)
4588             self.failUnlessReallyEqual(s["count-unknown"], 1)
4589         d.addCallback(_done)
4590
4591         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4592         def _check_manifest(res):
4593             self.failUnless(res.endswith("\n"))
4594             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4595             self.failUnlessReallyEqual(len(units), 5+1)
4596             self.failUnlessEqual(units[-1]["type"], "stats")
4597             first = units[0]
4598             self.failUnlessEqual(first["path"], [])
4599             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4600             self.failUnlessEqual(first["type"], "directory")
4601             stats = units[-1]["stats"]
4602             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4603             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4604             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4605             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4606             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4607         d.addCallback(_check_manifest)
4608
4609         # now add root/subdir and root/subdir/grandchild, then make subdir
4610         # unrecoverable, then see what happens
4611
4612         d.addCallback(lambda ign:
4613                       self.rootnode.create_subdirectory(u"subdir"))
4614         d.addCallback(_stash_uri, "subdir")
4615         d.addCallback(lambda subdir_node:
4616                       subdir_node.add_file(u"grandchild",
4617                                            upload.Data(DATA+"2",
4618                                                        convergence="")))
4619         d.addCallback(_stash_uri, "grandchild")
4620
4621         d.addCallback(lambda ign:
4622                       self.delete_shares_numbered(self.uris["subdir"],
4623                                                   range(1, 10)))
4624
4625         # root
4626         # root/good
4627         # root/small
4628         # root/sick
4629         # root/future
4630         # root/subdir [unrecoverable]
4631         # root/subdir/grandchild
4632
4633         # how should a streaming-JSON API indicate fatal error?
4634         # answer: emit ERROR: instead of a JSON string
4635
4636         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4637         def _check_broken_manifest(res):
4638             lines = res.splitlines()
4639             error_lines = [i
4640                            for (i,line) in enumerate(lines)
4641                            if line.startswith("ERROR:")]
4642             if not error_lines:
4643                 self.fail("no ERROR: in output: %s" % (res,))
4644             first_error = error_lines[0]
4645             error_line = lines[first_error]
4646             error_msg = lines[first_error+1:]
4647             error_msg_s = "\n".join(error_msg) + "\n"
4648             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4649                               error_line)
4650             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4651             units = [simplejson.loads(line) for line in lines[:first_error]]
4652             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4653             last_unit = units[-1]
4654             self.failUnlessEqual(last_unit["path"], ["subdir"])
4655         d.addCallback(_check_broken_manifest)
4656
4657         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4658         def _check_broken_deepcheck(res):
4659             lines = res.splitlines()
4660             error_lines = [i
4661                            for (i,line) in enumerate(lines)
4662                            if line.startswith("ERROR:")]
4663             if not error_lines:
4664                 self.fail("no ERROR: in output: %s" % (res,))
4665             first_error = error_lines[0]
4666             error_line = lines[first_error]
4667             error_msg = lines[first_error+1:]
4668             error_msg_s = "\n".join(error_msg) + "\n"
4669             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4670                               error_line)
4671             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4672             units = [simplejson.loads(line) for line in lines[:first_error]]
4673             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4674             last_unit = units[-1]
4675             self.failUnlessEqual(last_unit["path"], ["subdir"])
4676             r = last_unit["check-results"]["results"]
4677             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4678             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4679             self.failUnlessReallyEqual(r["recoverable"], False)
4680         d.addCallback(_check_broken_deepcheck)
4681
4682         d.addErrback(self.explain_web_error)
4683         return d
4684
4685     def test_deep_check_and_repair(self):
4686         self.basedir = "web/Grid/deep_check_and_repair"
4687         self.set_up_grid()
4688         c0 = self.g.clients[0]
4689         self.uris = {}
4690         self.fileurls = {}
4691         DATA = "data" * 100
4692         d = c0.create_dirnode()
4693         def _stash_root_and_create_file(n):
4694             self.rootnode = n
4695             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4696             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4697         d.addCallback(_stash_root_and_create_file)
4698         def _stash_uri(fn, which):
4699             self.uris[which] = fn.get_uri()
4700         d.addCallback(_stash_uri, "good")
4701         d.addCallback(lambda ign:
4702                       self.rootnode.add_file(u"small",
4703                                              upload.Data("literal",
4704                                                         convergence="")))
4705         d.addCallback(_stash_uri, "small")
4706         d.addCallback(lambda ign:
4707                       self.rootnode.add_file(u"sick",
4708                                              upload.Data(DATA+"1",
4709                                                         convergence="")))
4710         d.addCallback(_stash_uri, "sick")
4711         #d.addCallback(lambda ign:
4712         #              self.rootnode.add_file(u"dead",
4713         #                                     upload.Data(DATA+"2",
4714         #                                                convergence="")))
4715         #d.addCallback(_stash_uri, "dead")
4716
4717         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4718         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4719         #d.addCallback(_stash_uri, "corrupt")
4720
4721         def _clobber_shares(ignored):
4722             good_shares = self.find_uri_shares(self.uris["good"])
4723             self.failUnlessReallyEqual(len(good_shares), 10)
4724             sick_shares = self.find_uri_shares(self.uris["sick"])
4725             os.unlink(sick_shares[0][2])
4726             #dead_shares = self.find_uri_shares(self.uris["dead"])
4727             #for i in range(1, 10):
4728             #    os.unlink(dead_shares[i][2])
4729
4730             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4731             #cso = CorruptShareOptions()
4732             #cso.stdout = StringIO()
4733             #cso.parseOptions([c_shares[0][2]])
4734             #corrupt_share(cso)
4735         d.addCallback(_clobber_shares)
4736
4737         # root
4738         # root/good   CHK, 10 shares
4739         # root/small  LIT
4740         # root/sick   CHK, 9 shares
4741
4742         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4743         def _done(res):
4744             units = [simplejson.loads(line)
4745                      for line in res.splitlines()
4746                      if line]
4747             self.failUnlessReallyEqual(len(units), 4+1)
4748             # should be parent-first
4749             u0 = units[0]
4750             self.failUnlessEqual(u0["path"], [])
4751             self.failUnlessEqual(u0["type"], "directory")
4752             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4753             u0crr = u0["check-and-repair-results"]
4754             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4755             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4756
4757             ugood = [u for u in units
4758                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4759             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4760             ugoodcrr = ugood["check-and-repair-results"]
4761             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4762             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4763
4764             usick = [u for u in units
4765                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4766             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4767             usickcrr = usick["check-and-repair-results"]
4768             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4769             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4770             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4771             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4772
4773             stats = units[-1]
4774             self.failUnlessEqual(stats["type"], "stats")
4775             s = stats["stats"]
4776             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4777             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4778             self.failUnlessReallyEqual(s["count-directories"], 1)
4779         d.addCallback(_done)
4780
4781         d.addErrback(self.explain_web_error)
4782         return d
4783
4784     def _count_leases(self, ignored, which):
4785         u = self.uris[which]
4786         shares = self.find_uri_shares(u)
4787         lease_counts = []
4788         for shnum, serverid, fn in shares:
4789             sf = get_share_file(fn)
4790             num_leases = len(list(sf.get_leases()))
4791             lease_counts.append( (fn, num_leases) )
4792         return lease_counts
4793
4794     def _assert_leasecount(self, lease_counts, expected):
4795         for (fn, num_leases) in lease_counts:
4796             if num_leases != expected:
4797                 self.fail("expected %d leases, have %d, on %s" %
4798                           (expected, num_leases, fn))
4799
4800     def test_add_lease(self):
4801         self.basedir = "web/Grid/add_lease"
4802         self.set_up_grid(num_clients=2)
4803         c0 = self.g.clients[0]
4804         self.uris = {}
4805         DATA = "data" * 100
4806         d = c0.upload(upload.Data(DATA, convergence=""))
4807         def _stash_uri(ur, which):
4808             self.uris[which] = ur.uri
4809         d.addCallback(_stash_uri, "one")
4810         d.addCallback(lambda ign:
4811                       c0.upload(upload.Data(DATA+"1", convergence="")))
4812         d.addCallback(_stash_uri, "two")
4813         def _stash_mutable_uri(n, which):
4814             self.uris[which] = n.get_uri()
4815             assert isinstance(self.uris[which], str)
4816         d.addCallback(lambda ign:
4817             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4818         d.addCallback(_stash_mutable_uri, "mutable")
4819
4820         def _compute_fileurls(ignored):
4821             self.fileurls = {}
4822             for which in self.uris:
4823                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4824         d.addCallback(_compute_fileurls)
4825
4826         d.addCallback(self._count_leases, "one")
4827         d.addCallback(self._assert_leasecount, 1)
4828         d.addCallback(self._count_leases, "two")
4829         d.addCallback(self._assert_leasecount, 1)
4830         d.addCallback(self._count_leases, "mutable")
4831         d.addCallback(self._assert_leasecount, 1)
4832
4833         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4834         def _got_html_good(res):
4835             self.failUnless("Healthy" in res, res)
4836             self.failIf("Not Healthy" in res, res)
4837         d.addCallback(_got_html_good)
4838
4839         d.addCallback(self._count_leases, "one")
4840         d.addCallback(self._assert_leasecount, 1)
4841         d.addCallback(self._count_leases, "two")
4842         d.addCallback(self._assert_leasecount, 1)
4843         d.addCallback(self._count_leases, "mutable")
4844         d.addCallback(self._assert_leasecount, 1)
4845
4846         # this CHECK uses the original client, which uses the same
4847         # lease-secrets, so it will just renew the original lease
4848         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4849         d.addCallback(_got_html_good)
4850
4851         d.addCallback(self._count_leases, "one")
4852         d.addCallback(self._assert_leasecount, 1)
4853         d.addCallback(self._count_leases, "two")
4854         d.addCallback(self._assert_leasecount, 1)
4855         d.addCallback(self._count_leases, "mutable")
4856         d.addCallback(self._assert_leasecount, 1)
4857
4858         # this CHECK uses an alternate client, which adds a second lease
4859         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4860         d.addCallback(_got_html_good)
4861
4862         d.addCallback(self._count_leases, "one")
4863         d.addCallback(self._assert_leasecount, 2)
4864         d.addCallback(self._count_leases, "two")
4865         d.addCallback(self._assert_leasecount, 1)
4866         d.addCallback(self._count_leases, "mutable")
4867         d.addCallback(self._assert_leasecount, 1)
4868
4869         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4870         d.addCallback(_got_html_good)
4871
4872         d.addCallback(self._count_leases, "one")
4873         d.addCallback(self._assert_leasecount, 2)
4874         d.addCallback(self._count_leases, "two")
4875         d.addCallback(self._assert_leasecount, 1)
4876         d.addCallback(self._count_leases, "mutable")
4877         d.addCallback(self._assert_leasecount, 1)
4878
4879         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4880                       clientnum=1)
4881         d.addCallback(_got_html_good)
4882
4883         d.addCallback(self._count_leases, "one")
4884         d.addCallback(self._assert_leasecount, 2)
4885         d.addCallback(self._count_leases, "two")
4886         d.addCallback(self._assert_leasecount, 1)
4887         d.addCallback(self._count_leases, "mutable")
4888         d.addCallback(self._assert_leasecount, 2)
4889
4890         d.addErrback(self.explain_web_error)
4891         return d
4892
4893     def test_deep_add_lease(self):
4894         self.basedir = "web/Grid/deep_add_lease"
4895         self.set_up_grid(num_clients=2)
4896         c0 = self.g.clients[0]
4897         self.uris = {}
4898         self.fileurls = {}
4899         DATA = "data" * 100
4900         d = c0.create_dirnode()
4901         def _stash_root_and_create_file(n):
4902             self.rootnode = n
4903             self.uris["root"] = n.get_uri()
4904             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4905             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4906         d.addCallback(_stash_root_and_create_file)
4907         def _stash_uri(fn, which):
4908             self.uris[which] = fn.get_uri()
4909         d.addCallback(_stash_uri, "one")
4910         d.addCallback(lambda ign:
4911                       self.rootnode.add_file(u"small",
4912                                              upload.Data("literal",
4913                                                         convergence="")))
4914         d.addCallback(_stash_uri, "small")
4915
4916         d.addCallback(lambda ign:
4917             c0.create_mutable_file(publish.MutableData("mutable")))
4918         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4919         d.addCallback(_stash_uri, "mutable")
4920
4921         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4922         def _done(res):
4923             units = [simplejson.loads(line)
4924                      for line in res.splitlines()
4925                      if line]
4926             # root, one, small, mutable,   stats
4927             self.failUnlessReallyEqual(len(units), 4+1)
4928         d.addCallback(_done)
4929
4930         d.addCallback(self._count_leases, "root")
4931         d.addCallback(self._assert_leasecount, 1)
4932         d.addCallback(self._count_leases, "one")
4933         d.addCallback(self._assert_leasecount, 1)
4934         d.addCallback(self._count_leases, "mutable")
4935         d.addCallback(self._assert_leasecount, 1)
4936
4937         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4938         d.addCallback(_done)
4939
4940         d.addCallback(self._count_leases, "root")
4941         d.addCallback(self._assert_leasecount, 1)
4942         d.addCallback(self._count_leases, "one")
4943         d.addCallback(self._assert_leasecount, 1)
4944         d.addCallback(self._count_leases, "mutable")
4945         d.addCallback(self._assert_leasecount, 1)
4946
4947         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4948                       clientnum=1)
4949         d.addCallback(_done)
4950
4951         d.addCallback(self._count_leases, "root")
4952         d.addCallback(self._assert_leasecount, 2)
4953         d.addCallback(self._count_leases, "one")
4954         d.addCallback(self._assert_leasecount, 2)
4955         d.addCallback(self._count_leases, "mutable")
4956         d.addCallback(self._assert_leasecount, 2)
4957
4958         d.addErrback(self.explain_web_error)
4959         return d
4960
4961
4962     def test_exceptions(self):
4963         self.basedir = "web/Grid/exceptions"
4964         self.set_up_grid(num_clients=1, num_servers=2)
4965         c0 = self.g.clients[0]
4966         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4967         self.fileurls = {}
4968         DATA = "data" * 100
4969         d = c0.create_dirnode()
4970         def _stash_root(n):
4971             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4972             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4973             return n
4974         d.addCallback(_stash_root)
4975         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4976         def _stash_bad(ur):
4977             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4978             self.delete_shares_numbered(ur.uri, range(1,10))
4979
4980             u = uri.from_string(ur.uri)
4981             u.key = testutil.flip_bit(u.key, 0)
4982             baduri = u.to_string()
4983             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4984         d.addCallback(_stash_bad)
4985         d.addCallback(lambda ign: c0.create_dirnode())
4986         def _mangle_dirnode_1share(n):
4987             u = n.get_uri()
4988             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4989             self.fileurls["dir-1share-json"] = url + "?t=json"
4990             self.delete_shares_numbered(u, range(1,10))
4991         d.addCallback(_mangle_dirnode_1share)
4992         d.addCallback(lambda ign: c0.create_dirnode())
4993         def _mangle_dirnode_0share(n):
4994             u = n.get_uri()
4995             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4996             self.fileurls["dir-0share-json"] = url + "?t=json"
4997             self.delete_shares_numbered(u, range(0,10))
4998         d.addCallback(_mangle_dirnode_0share)
4999
5000         # NotEnoughSharesError should be reported sensibly, with a
5001         # text/plain explanation of the problem, and perhaps some
5002         # information on which shares *could* be found.
5003
5004         d.addCallback(lambda ignored:
5005                       self.shouldHTTPError("GET unrecoverable",
5006                                            410, "Gone", "NoSharesError",
5007                                            self.GET, self.fileurls["0shares"]))
5008         def _check_zero_shares(body):
5009             self.failIf("<html>" in body, body)
5010             body = " ".join(body.strip().split())
5011             exp = ("NoSharesError: no shares could be found. "
5012                    "Zero shares usually indicates a corrupt URI, or that "
5013                    "no servers were connected, but it might also indicate "
5014                    "severe corruption. You should perform a filecheck on "
5015                    "this object to learn more. The full error message is: "
5016                    "no shares (need 3). Last failure: None")
5017             self.failUnlessReallyEqual(exp, body)
5018         d.addCallback(_check_zero_shares)
5019
5020
5021         d.addCallback(lambda ignored:
5022                       self.shouldHTTPError("GET 1share",
5023                                            410, "Gone", "NotEnoughSharesError",
5024                                            self.GET, self.fileurls["1share"]))
5025         def _check_one_share(body):
5026             self.failIf("<html>" in body, body)
5027             body = " ".join(body.strip().split())
5028             msgbase = ("NotEnoughSharesError: This indicates that some "
5029                        "servers were unavailable, or that shares have been "
5030                        "lost to server departure, hard drive failure, or disk "
5031                        "corruption. You should perform a filecheck on "
5032                        "this object to learn more. The full error message is:"
5033                        )
5034             msg1 = msgbase + (" ran out of shares:"
5035                               " complete=sh0"
5036                               " pending="
5037                               " overdue= unused= need 3. Last failure: None")
5038             msg2 = msgbase + (" ran out of shares:"
5039                               " complete="
5040                               " pending=Share(sh0-on-xgru5)"
5041                               " overdue= unused= need 3. Last failure: None")
5042             self.failUnless(body == msg1 or body == msg2, body)
5043         d.addCallback(_check_one_share)
5044
5045         d.addCallback(lambda ignored:
5046                       self.shouldHTTPError("GET imaginary",
5047                                            404, "Not Found", None,
5048                                            self.GET, self.fileurls["imaginary"]))
5049         def _missing_child(body):
5050             self.failUnless("No such child: imaginary" in body, body)
5051         d.addCallback(_missing_child)
5052
5053         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5054         def _check_0shares_dir_html(body):
5055             self.failUnless("<html>" in body, body)
5056             # we should see the regular page, but without the child table or
5057             # the dirops forms
5058             body = " ".join(body.strip().split())
5059             self.failUnlessIn('href="?t=info">More info on this directory',
5060                               body)
5061             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5062                    "could not be retrieved, because there were insufficient "
5063                    "good shares. This might indicate that no servers were "
5064                    "connected, insufficient servers were connected, the URI "
5065                    "was corrupt, or that shares have been lost due to server "
5066                    "departure, hard drive failure, or disk corruption. You "
5067                    "should perform a filecheck on this object to learn more.")
5068             self.failUnlessIn(exp, body)
5069             self.failUnlessIn("No upload forms: directory is unreadable", body)
5070         d.addCallback(_check_0shares_dir_html)
5071
5072         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5073         def _check_1shares_dir_html(body):
5074             # at some point, we'll split UnrecoverableFileError into 0-shares
5075             # and some-shares like we did for immutable files (since there
5076             # are different sorts of advice to offer in each case). For now,
5077             # they present the same way.
5078             self.failUnless("<html>" in body, body)
5079             body = " ".join(body.strip().split())
5080             self.failUnlessIn('href="?t=info">More info on this directory',
5081                               body)
5082             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5083                    "could not be retrieved, because there were insufficient "
5084                    "good shares. This might indicate that no servers were "
5085                    "connected, insufficient servers were connected, the URI "
5086                    "was corrupt, or that shares have been lost due to server "
5087                    "departure, hard drive failure, or disk corruption. You "
5088                    "should perform a filecheck on this object to learn more.")
5089             self.failUnlessIn(exp, body)
5090             self.failUnlessIn("No upload forms: directory is unreadable", body)
5091         d.addCallback(_check_1shares_dir_html)
5092
5093         d.addCallback(lambda ignored:
5094                       self.shouldHTTPError("GET dir-0share-json",
5095                                            410, "Gone", "UnrecoverableFileError",
5096                                            self.GET,
5097                                            self.fileurls["dir-0share-json"]))
5098         def _check_unrecoverable_file(body):
5099             self.failIf("<html>" in body, body)
5100             body = " ".join(body.strip().split())
5101             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5102                    "could not be retrieved, because there were insufficient "
5103                    "good shares. This might indicate that no servers were "
5104                    "connected, insufficient servers were connected, the URI "
5105                    "was corrupt, or that shares have been lost due to server "
5106                    "departure, hard drive failure, or disk corruption. You "
5107                    "should perform a filecheck on this object to learn more.")
5108             self.failUnlessReallyEqual(exp, body)
5109         d.addCallback(_check_unrecoverable_file)
5110
5111         d.addCallback(lambda ignored:
5112                       self.shouldHTTPError("GET dir-1share-json",
5113                                            410, "Gone", "UnrecoverableFileError",
5114                                            self.GET,
5115                                            self.fileurls["dir-1share-json"]))
5116         d.addCallback(_check_unrecoverable_file)
5117
5118         d.addCallback(lambda ignored:
5119                       self.shouldHTTPError("GET imaginary",
5120                                            404, "Not Found", None,
5121                                            self.GET, self.fileurls["imaginary"]))
5122
5123         # attach a webapi child that throws a random error, to test how it
5124         # gets rendered.
5125         w = c0.getServiceNamed("webish")
5126         w.root.putChild("ERRORBOOM", ErrorBoom())
5127
5128         # "Accept: */*" :        should get a text/html stack trace
5129         # "Accept: text/plain" : should get a text/plain stack trace
5130         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5131         # no Accept header:      should get a text/html stack trace
5132
5133         d.addCallback(lambda ignored:
5134                       self.shouldHTTPError("GET errorboom_html",
5135                                            500, "Internal Server Error", None,
5136                                            self.GET, "ERRORBOOM",
5137                                            headers={"accept": ["*/*"]}))
5138         def _internal_error_html1(body):
5139             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5140         d.addCallback(_internal_error_html1)
5141
5142         d.addCallback(lambda ignored:
5143                       self.shouldHTTPError("GET errorboom_text",
5144                                            500, "Internal Server Error", None,
5145                                            self.GET, "ERRORBOOM",
5146                                            headers={"accept": ["text/plain"]}))
5147         def _internal_error_text2(body):
5148             self.failIf("<html>" in body, body)
5149             self.failUnless(body.startswith("Traceback "), body)
5150         d.addCallback(_internal_error_text2)
5151
5152         CLI_accepts = "text/plain, application/octet-stream"
5153         d.addCallback(lambda ignored:
5154                       self.shouldHTTPError("GET errorboom_text",
5155                                            500, "Internal Server Error", None,
5156                                            self.GET, "ERRORBOOM",
5157                                            headers={"accept": [CLI_accepts]}))
5158         def _internal_error_text3(body):
5159             self.failIf("<html>" in body, body)
5160             self.failUnless(body.startswith("Traceback "), body)
5161         d.addCallback(_internal_error_text3)
5162
5163         d.addCallback(lambda ignored:
5164                       self.shouldHTTPError("GET errorboom_text",
5165                                            500, "Internal Server Error", None,
5166                                            self.GET, "ERRORBOOM"))
5167         def _internal_error_html4(body):
5168             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5169         d.addCallback(_internal_error_html4)
5170
5171         def _flush_errors(res):
5172             # Trial: please ignore the CompletelyUnhandledError in the logs
5173             self.flushLoggedErrors(CompletelyUnhandledError)
5174             return res
5175         d.addBoth(_flush_errors)
5176
5177         return d
5178
5179     def test_blacklist(self):
5180         # download from a blacklisted URI, get an error
5181         self.basedir = "web/Grid/blacklist"
5182         self.set_up_grid()
5183         c0 = self.g.clients[0]
5184         c0_basedir = c0.basedir
5185         fn = os.path.join(c0_basedir, "access.blacklist")
5186         self.uris = {}
5187         DATA = "off-limits " * 50
5188
5189         d = c0.upload(upload.Data(DATA, convergence=""))
5190         def _stash_uri_and_create_dir(ur):
5191             self.uri = ur.uri
5192             self.url = "uri/"+self.uri
5193             u = uri.from_string_filenode(self.uri)
5194             self.si = u.get_storage_index()
5195             childnode = c0.create_node_from_uri(self.uri, None)
5196             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5197         d.addCallback(_stash_uri_and_create_dir)
5198         def _stash_dir(node):
5199             self.dir_node = node
5200             self.dir_uri = node.get_uri()
5201             self.dir_url = "uri/"+self.dir_uri
5202         d.addCallback(_stash_dir)
5203         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5204         def _check_dir_html(body):
5205             self.failUnlessIn("<html>", body)
5206             self.failUnlessIn("blacklisted.txt</a>", body)
5207         d.addCallback(_check_dir_html)
5208         d.addCallback(lambda ign: self.GET(self.url))
5209         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5210
5211         def _blacklist(ign):
5212             f = open(fn, "w")
5213             f.write(" # this is a comment\n")
5214             f.write(" \n")
5215             f.write("\n") # also exercise blank lines
5216             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5217             f.close()
5218             # clients should be checking the blacklist each time, so we don't
5219             # need to restart the client
5220         d.addCallback(_blacklist)
5221         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5222                                                        403, "Forbidden",
5223                                                        "Access Prohibited: off-limits",
5224                                                        self.GET, self.url))
5225
5226         # We should still be able to list the parent directory, in HTML...
5227         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5228         def _check_dir_html2(body):
5229             self.failUnlessIn("<html>", body)
5230             self.failUnlessIn("blacklisted.txt</strike>", body)
5231         d.addCallback(_check_dir_html2)
5232
5233         # ... and in JSON (used by CLI).
5234         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5235         def _check_dir_json(res):
5236             data = simplejson.loads(res)
5237             self.failUnless(isinstance(data, list), data)
5238             self.failUnlessEqual(data[0], "dirnode")
5239             self.failUnless(isinstance(data[1], dict), data)
5240             self.failUnlessIn("children", data[1])
5241             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5242             childdata = data[1]["children"]["blacklisted.txt"]
5243             self.failUnless(isinstance(childdata, list), data)
5244             self.failUnlessEqual(childdata[0], "filenode")
5245             self.failUnless(isinstance(childdata[1], dict), data)
5246         d.addCallback(_check_dir_json)
5247
5248         def _unblacklist(ign):
5249             open(fn, "w").close()
5250             # the Blacklist object watches mtime to tell when the file has
5251             # changed, but on windows this test will run faster than the
5252             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5253             # to force a reload.
5254             self.g.clients[0].blacklist.last_mtime -= 2.0
5255         d.addCallback(_unblacklist)
5256
5257         # now a read should work
5258         d.addCallback(lambda ign: self.GET(self.url))
5259         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5260
5261         # read again to exercise the blacklist-is-unchanged logic
5262         d.addCallback(lambda ign: self.GET(self.url))
5263         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5264
5265         # now add a blacklisted directory, and make sure files under it are
5266         # refused too
5267         def _add_dir(ign):
5268             childnode = c0.create_node_from_uri(self.uri, None)
5269             return c0.create_dirnode({u"child": (childnode,{}) })
5270         d.addCallback(_add_dir)
5271         def _get_dircap(dn):
5272             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5273             self.dir_url_base = "uri/"+dn.get_write_uri()
5274             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5275             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5276             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5277             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5278         d.addCallback(_get_dircap)
5279         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5280         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5281         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5282         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5283         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5284         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5285         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5286         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5287         d.addCallback(lambda ign: self.GET(self.child_url))
5288         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5289
5290         def _block_dir(ign):
5291             f = open(fn, "w")
5292             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5293             f.close()
5294             self.g.clients[0].blacklist.last_mtime -= 2.0
5295         d.addCallback(_block_dir)
5296         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5297                                                        403, "Forbidden",
5298                                                        "Access Prohibited: dir-off-limits",
5299                                                        self.GET, self.dir_url_base))
5300         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5301                                                        403, "Forbidden",
5302                                                        "Access Prohibited: dir-off-limits",
5303                                                        self.GET, self.dir_url_json1))
5304         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5305                                                        403, "Forbidden",
5306                                                        "Access Prohibited: dir-off-limits",
5307                                                        self.GET, self.dir_url_json2))
5308         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5309                                                        403, "Forbidden",
5310                                                        "Access Prohibited: dir-off-limits",
5311                                                        self.GET, self.dir_url_json_ro))
5312         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5313                                                        403, "Forbidden",
5314                                                        "Access Prohibited: dir-off-limits",
5315                                                        self.GET, self.child_url))
5316         return d
5317
5318
5319 class CompletelyUnhandledError(Exception):
5320     pass
5321 class ErrorBoom(rend.Page):
5322     def beforeRender(self, ctx):
5323         raise CompletelyUnhandledError("whoops")