src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649         d.addCallback(_check_dl_json)
 650         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 651         def _check_ul(res):
 652             self.failUnless("File Upload Status" in res, res)
 653         d.addCallback(_check_ul)
 654         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 655         def _check_mapupdate(res):
 656             self.failUnless("Mutable File Servermap Update Status" in res, res)
 657         d.addCallback(_check_mapupdate)
 658         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 659         def _check_publish(res):
 660             self.failUnless("Mutable File Publish Status" in res, res)
 661         d.addCallback(_check_publish)
 662         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 663         def _check_retrieve(res):
 664             self.failUnless("Mutable File Retrieve Status" in res, res)
 665         d.addCallback(_check_retrieve)
 666
 667         return d
 668
 669     def test_status_numbers(self):
 670         drrm = status.DownloadResultsRendererMixin()
 671         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 680
 681         urrm = status.UploadResultsRendererMixin()
 682         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 691
 692     def test_GET_FILEURL(self):
 693         d = self.GET(self.public_url + "/foo/bar.txt")
 694         d.addCallback(self.failUnlessIsBarDotTxt)
 695         return d
 696
 697     def test_GET_FILEURL_range(self):
 698         headers = {"range": "bytes=1-10"}
 699         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 700                      return_response=True)
 701         def _got((res, status, headers)):
 702             self.failUnlessReallyEqual(int(status), 206)
 703             self.failUnless(headers.has_key("content-range"))
 704             self.failUnlessReallyEqual(headers["content-range"][0],
 705                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 706             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 707         d.addCallback(_got)
 708         return d
 709
 710     def test_GET_FILEURL_partial_range(self):
 711         headers = {"range": "bytes=5-"}
 712         length  = len(self.BAR_CONTENTS)
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 206)
 717             self.failUnless(headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(headers["content-range"][0],
 719                                        "bytes 5-%d/%d" % (length-1, length))
 720             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 721         d.addCallback(_got)
 722         return d
 723
 724     def test_GET_FILEURL_partial_end_range(self):
 725         headers = {"range": "bytes=-5"}
 726         length  = len(self.BAR_CONTENTS)
 727         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 728                      return_response=True)
 729         def _got((res, status, headers)):
 730             self.failUnlessReallyEqual(int(status), 206)
 731             self.failUnless(headers.has_key("content-range"))
 732             self.failUnlessReallyEqual(headers["content-range"][0],
 733                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 734             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_partial_range_overrun(self):
 739         headers = {"range": "bytes=100-200"}
 740         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 741                              "416 Requested Range not satisfiable",
 742                              "First beyond end of file",
 743                              self.GET, self.public_url + "/foo/bar.txt",
 744                              headers=headers)
 745         return d
 746
 747     def test_HEAD_FILEURL_range(self):
 748         headers = {"range": "bytes=1-10"}
 749         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 750                      return_response=True)
 751         def _got((res, status, headers)):
 752             self.failUnlessReallyEqual(res, "")
 753             self.failUnlessReallyEqual(int(status), 206)
 754             self.failUnless(headers.has_key("content-range"))
 755             self.failUnlessReallyEqual(headers["content-range"][0],
 756                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_HEAD_FILEURL_partial_range(self):
 761         headers = {"range": "bytes=5-"}
 762         length  = len(self.BAR_CONTENTS)
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 764                      return_response=True)
 765         def _got((res, status, headers)):
 766             self.failUnlessReallyEqual(int(status), 206)
 767             self.failUnless(headers.has_key("content-range"))
 768             self.failUnlessReallyEqual(headers["content-range"][0],
 769                                        "bytes 5-%d/%d" % (length-1, length))
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_HEAD_FILEURL_partial_end_range(self):
 774         headers = {"range": "bytes=-5"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range_overrun(self):
 787         headers = {"range": "bytes=100-200"}
 788         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 789                              "416 Requested Range not satisfiable",
 790                              "",
 791                              self.HEAD, self.public_url + "/foo/bar.txt",
 792                              headers=headers)
 793         return d
 794
 795     def test_GET_FILEURL_range_bad(self):
 796         headers = {"range": "BOGUS=fizbop-quarnak"}
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 200)
 801             self.failUnless(not headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 803         d.addCallback(_got)
 804         return d
 805
 806     def test_HEAD_FILEURL(self):
 807         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 808         def _got((res, status, headers)):
 809             self.failUnlessReallyEqual(res, "")
 810             self.failUnlessReallyEqual(headers["content-length"][0],
 811                                        str(len(self.BAR_CONTENTS)))
 812             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 813         d.addCallback(_got)
 814         return d
 815
 816     def test_GET_FILEURL_named(self):
 817         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 818         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 819         d = self.GET(base + "/@@name=/blah.txt")
 820         d.addCallback(self.failUnlessIsBarDotTxt)
 821         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 822         d.addCallback(self.failUnlessIsBarDotTxt)
 823         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 824         d.addCallback(self.failUnlessIsBarDotTxt)
 825         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 826         d.addCallback(self.failUnlessIsBarDotTxt)
 827         save_url = base + "?save=true&filename=blah.txt"
 828         d.addCallback(lambda res: self.GET(save_url))
 829         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 830         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 831         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 832         u_url = base + "?save=true&filename=" + u_fn_e
 833         d.addCallback(lambda res: self.GET(u_url))
 834         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 835         return d
 836
 837     def test_PUT_FILEURL_named_bad(self):
 838         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 839         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 840                              "400 Bad Request",
 841                              "/file can only be used with GET or HEAD",
 842                              self.PUT, base + "/@@name=/blah.txt", "")
 843         return d
 844
 845
 846     def test_GET_DIRURL_named_bad(self):
 847         base = "/file/%s" % urllib.quote(self._foo_uri)
 848         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 849                              "400 Bad Request",
 850                              "is not a file-cap",
 851                              self.GET, base + "/@@name=/blah.txt")
 852         return d
 853
 854     def test_GET_slash_file_bad(self):
 855         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 856                              "404 Not Found",
 857                              "/file must be followed by a file-cap and a name",
 858                              self.GET, "/file")
 859         return d
 860
 861     def test_GET_unhandled_URI_named(self):
 862         contents, n, newuri = self.makefile(12)
 863         verifier_cap = n.get_verify_cap().to_string()
 864         base = "/file/%s" % urllib.quote(verifier_cap)
 865         # client.create_node_from_uri() can't handle verify-caps
 866         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 867                              "400 Bad Request", "is not a file-cap",
 868                              self.GET, base)
 869         return d
 870
 871     def test_GET_unhandled_URI(self):
 872         contents, n, newuri = self.makefile(12)
 873         verifier_cap = n.get_verify_cap().to_string()
 874         base = "/uri/%s" % urllib.quote(verifier_cap)
 875         # client.create_node_from_uri() can't handle verify-caps
 876         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 877                              "400 Bad Request",
 878                              "GET unknown URI type: can only do t=info",
 879                              self.GET, base)
 880         return d
 881
 882     def test_GET_FILE_URI(self):
 883         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 884         d = self.GET(base)
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         return d
 887
 888     def test_GET_FILE_URI_mdmf(self):
 889         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 890         d = self.GET(base)
 891         d.addCallback(self.failUnlessIsQuuxDotTxt)
 892         return d
 893
 894     def test_GET_FILE_URI_mdmf_extensions(self):
 895         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 896         d = self.GET(base)
 897         d.addCallback(self.failUnlessIsQuuxDotTxt)
 898         return d
 899
 900     def test_GET_FILE_URI_mdmf_readonly(self):
 901         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 902         d = self.GET(base)
 903         d.addCallback(self.failUnlessIsQuuxDotTxt)
 904         return d
 905
 906     def test_GET_FILE_URI_badchild(self):
 907         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 908         errmsg = "Files have no children, certainly not named 'boguschild'"
 909         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 910                              "400 Bad Request", errmsg,
 911                              self.GET, base)
 912         return d
 913
 914     def test_PUT_FILE_URI_badchild(self):
 915         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 916         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 917         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 918                              "400 Bad Request", errmsg,
 919                              self.PUT, base, "")
 920         return d
 921
 922     def test_PUT_FILE_URI_mdmf(self):
 923         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 924         self._quux_new_contents = "new_contents"
 925         d = self.GET(base)
 926         d.addCallback(lambda res:
 927             self.failUnlessIsQuuxDotTxt(res))
 928         d.addCallback(lambda ignored:
 929             self.PUT(base, self._quux_new_contents))
 930         d.addCallback(lambda ignored:
 931             self.GET(base))
 932         d.addCallback(lambda res:
 933             self.failUnlessReallyEqual(res, self._quux_new_contents))
 934         return d
 935
 936     def test_PUT_FILE_URI_mdmf_extensions(self):
 937         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 938         self._quux_new_contents = "new_contents"
 939         d = self.GET(base)
 940         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 941         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 942         d.addCallback(lambda ignored: self.GET(base))
 943         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 944                                                        res))
 945         return d
 946
 947     def test_PUT_FILE_URI_mdmf_readonly(self):
 948         # We're not allowed to PUT things to a readonly cap.
 949         base = "/uri/%s" % self._quux_txt_readonly_uri
 950         d = self.GET(base)
 951         d.addCallback(lambda res:
 952             self.failUnlessIsQuuxDotTxt(res))
 953         # What should we get here? We get a 500 error now; that's not right.
 954         d.addCallback(lambda ignored:
 955             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 956                              "400 Bad Request", "read-only cap",
 957                              self.PUT, base, "new data"))
 958         return d
 959
 960     def test_PUT_FILE_URI_sdmf_readonly(self):
 961         # We're not allowed to put things to a readonly cap.
 962         base = "/uri/%s" % self._baz_txt_readonly_uri
 963         d = self.GET(base)
 964         d.addCallback(lambda res:
 965             self.failUnlessIsBazDotTxt(res))
 966         d.addCallback(lambda ignored:
 967             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 968                              "400 Bad Request", "read-only cap",
 969                              self.PUT, base, "new_data"))
 970         return d
 971
 972     # TODO: version of this with a Unicode filename
 973     def test_GET_FILEURL_save(self):
 974         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 975                      return_response=True)
 976         def _got((res, statuscode, headers)):
 977             content_disposition = headers["content-disposition"][0]
 978             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 979             self.failUnlessIsBarDotTxt(res)
 980         d.addCallback(_got)
 981         return d
 982
 983     def test_GET_FILEURL_missing(self):
 984         d = self.GET(self.public_url + "/foo/missing")
 985         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 986         return d
 987
 988     def test_GET_FILEURL_info_mdmf(self):
 989         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 990         def _got(res):
 991             self.failUnlessIn("mutable file (mdmf)", res)
 992             self.failUnlessIn(self._quux_txt_uri, res)
 993             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 994         d.addCallback(_got)
 995         return d
 996
 997     def test_GET_FILEURL_info_mdmf_readonly(self):
 998         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 999         def _got(res):
1000             self.failUnlessIn("mutable file (mdmf)", res)
1001             self.failIfIn(self._quux_txt_uri, res)
1002             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1003         d.addCallback(_got)
1004         return d
1005
1006     def test_GET_FILEURL_info_sdmf(self):
1007         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1008         def _got(res):
1009             self.failUnlessIn("mutable file (sdmf)", res)
1010             self.failUnlessIn(self._baz_txt_uri, res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_info_mdmf_extensions(self):
1015         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1016         def _got(res):
1017             self.failUnlessIn("mutable file (mdmf)", res)
1018             self.failUnlessIn(self._quux_txt_uri, res)
1019             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_PUT_overwrite_only_files(self):
1024         # create a directory, put a file in that directory.
1025         contents, n, filecap = self.makefile(8)
1026         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1027         d.addCallback(lambda res:
1028             self.PUT(self.public_url + "/foo/dir/file1.txt",
1029                      self.NEWFILE_CONTENTS))
1030         # try to overwrite the file with replace=only-files
1031         # (this should work)
1032         d.addCallback(lambda res:
1033             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1034                      filecap))
1035         d.addCallback(lambda res:
1036             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1037                  "There was already a child by that name, and you asked me "
1038                  "to not replace it",
1039                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1040                  filecap))
1041         return d
1042
1043     def test_PUT_NEWFILEURL(self):
1044         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1045         # TODO: we lose the response code, so we can't check this
1046         #self.failUnlessReallyEqual(responsecode, 201)
1047         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1048         d.addCallback(lambda res:
1049                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1050                                                       self.NEWFILE_CONTENTS))
1051         return d
1052
1053     def test_PUT_NEWFILEURL_not_mutable(self):
1054         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1055                      self.NEWFILE_CONTENTS)
1056         # TODO: we lose the response code, so we can't check this
1057         #self.failUnlessReallyEqual(responsecode, 201)
1058         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1059         d.addCallback(lambda res:
1060                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1061                                                       self.NEWFILE_CONTENTS))
1062         return d
1063
1064     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1065         # this should get us a few segments of an MDMF mutable file,
1066         # which we can then test for.
1067         contents = self.NEWFILE_CONTENTS * 300000
1068         d = self.PUT("/uri?format=mdmf",
1069                      contents)
1070         def _got_filecap(filecap):
1071             self.failUnless(filecap.startswith("URI:MDMF"))
1072             return filecap
1073         d.addCallback(_got_filecap)
1074         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1075         d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1076         return d
1077
1078     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1079         contents = self.NEWFILE_CONTENTS * 300000
1080         d = self.PUT("/uri?format=sdmf",
1081                      contents)
1082         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1083         d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1084         return d
1085
1086     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1087         contents = self.NEWFILE_CONTENTS * 300000
1088         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1089                                     400, "Bad Request", "Unknown format: foo",
1090                                     self.PUT, "/uri?format=foo",
1091                                     contents)
1092
1093     def test_PUT_NEWFILEURL_range_bad(self):
1094         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1095         target = self.public_url + "/foo/new.txt"
1096         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1097                              "501 Not Implemented",
1098                              "Content-Range in PUT not yet supported",
1099                              # (and certainly not for immutable files)
1100                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1101                              headers=headers)
1102         d.addCallback(lambda res:
1103                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1104         return d
1105
1106     def test_PUT_NEWFILEURL_mutable(self):
1107         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1108                      self.NEWFILE_CONTENTS)
1109         # TODO: we lose the response code, so we can't check this
1110         #self.failUnlessReallyEqual(responsecode, 201)
1111         def _check_uri(res):
1112             u = uri.from_string_mutable_filenode(res)
1113             self.failUnless(u.is_mutable())
1114             self.failIf(u.is_readonly())
1115             return res
1116         d.addCallback(_check_uri)
1117         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1118         d.addCallback(lambda res:
1119                       self.failUnlessMutableChildContentsAre(self._foo_node,
1120                                                              u"new.txt",
1121                                                              self.NEWFILE_CONTENTS))
1122         return d
1123
1124     def test_PUT_NEWFILEURL_mutable_toobig(self):
1125         # It is okay to upload large mutable files, so we should be able
1126         # to do that.
1127         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1128                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1129         return d
1130
1131     def test_PUT_NEWFILEURL_replace(self):
1132         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1133         # TODO: we lose the response code, so we can't check this
1134         #self.failUnlessReallyEqual(responsecode, 200)
1135         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1136         d.addCallback(lambda res:
1137                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1138                                                       self.NEWFILE_CONTENTS))
1139         return d
1140
1141     def test_PUT_NEWFILEURL_bad_t(self):
1142         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1143                              "PUT to a file: bad t=bogus",
1144                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1145                              "contents")
1146         return d
1147
1148     def test_PUT_NEWFILEURL_no_replace(self):
1149         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1150                      self.NEWFILE_CONTENTS)
1151         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1152                   "409 Conflict",
1153                   "There was already a child by that name, and you asked me "
1154                   "to not replace it")
1155         return d
1156
1157     def test_PUT_NEWFILEURL_mkdirs(self):
1158         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1159         fn = self._foo_node
1160         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1161         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1162         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1163         d.addCallback(lambda res:
1164                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1165                                                       self.NEWFILE_CONTENTS))
1166         return d
1167
1168     def test_PUT_NEWFILEURL_blocked(self):
1169         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1170                      self.NEWFILE_CONTENTS)
1171         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1172                   "409 Conflict",
1173                   "Unable to create directory 'blockingfile': a file was in the way")
1174         return d
1175
1176     def test_PUT_NEWFILEURL_emptyname(self):
1177         # an empty pathname component (i.e. a double-slash) is disallowed
1178         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1179                              "400 Bad Request",
1180                              "The webapi does not allow empty pathname components",
1181                              self.PUT, self.public_url + "/foo//new.txt", "")
1182         return d
1183
1184     def test_DELETE_FILEURL(self):
1185         d = self.DELETE(self.public_url + "/foo/bar.txt")
1186         d.addCallback(lambda res:
1187                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1188         return d
1189
1190     def test_DELETE_FILEURL_missing(self):
1191         d = self.DELETE(self.public_url + "/foo/missing")
1192         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1193         return d
1194
1195     def test_DELETE_FILEURL_missing2(self):
1196         d = self.DELETE(self.public_url + "/missing/missing")
1197         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1198         return d
1199
1200     def failUnlessHasBarDotTxtMetadata(self, res):
1201         data = simplejson.loads(res)
1202         self.failUnless(isinstance(data, list))
1203         self.failUnlessIn("metadata", data[1])
1204         self.failUnlessIn("tahoe", data[1]["metadata"])
1205         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1206         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1207         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1208                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1209
1210     def test_GET_FILEURL_json(self):
1211         # twisted.web.http.parse_qs ignores any query args without an '=', so
1212         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1213         # instead. This may make it tricky to emulate the S3 interface
1214         # completely.
1215         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1216         def _check1(data):
1217             self.failUnlessIsBarJSON(data)
1218             self.failUnlessHasBarDotTxtMetadata(data)
1219             return
1220         d.addCallback(_check1)
1221         return d
1222
1223     def test_GET_FILEURL_json_mutable_type(self):
1224         # The JSON should include format, which says whether the
1225         # file is SDMF or MDMF
1226         d = self.PUT("/uri?format=mdmf",
1227                      self.NEWFILE_CONTENTS * 300000)
1228         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1229         def _got_json(json, version):
1230             data = simplejson.loads(json)
1231             assert "filenode" == data[0]
1232             data = data[1]
1233             assert isinstance(data, dict)
1234
1235             self.failUnlessIn("format", data)
1236             self.failUnlessEqual(data["format"], version)
1237
1238         d.addCallback(_got_json, "mdmf")
1239         # Now make an SDMF file and check that it is reported correctly.
1240         d.addCallback(lambda ignored:
1241             self.PUT("/uri?format=sdmf",
1242                       self.NEWFILE_CONTENTS * 300000))
1243         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244         d.addCallback(_got_json, "sdmf")
1245         return d
1246
1247     def test_GET_FILEURL_json_mdmf(self):
1248         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1249         d.addCallback(self.failUnlessIsQuuxJSON)
1250         return d
1251
1252     def test_GET_FILEURL_json_missing(self):
1253         d = self.GET(self.public_url + "/foo/missing?json")
1254         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1255         return d
1256
1257     def test_GET_FILEURL_uri(self):
1258         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1259         def _check(res):
1260             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1261         d.addCallback(_check)
1262         d.addCallback(lambda res:
1263                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1264         def _check2(res):
1265             # for now, for files, uris and readonly-uris are the same
1266             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1267         d.addCallback(_check2)
1268         return d
1269
1270     def test_GET_FILEURL_badtype(self):
1271         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1272                                  "bad t=bogus",
1273                                  self.GET,
1274                                  self.public_url + "/foo/bar.txt?t=bogus")
1275         return d
1276
1277     def test_CSS_FILE(self):
1278         d = self.GET("/tahoe_css", followRedirect=True)
1279         def _check(res):
1280             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1281             self.failUnless(CSS_STYLE.search(res), res)
1282         d.addCallback(_check)
1283         return d
1284
1285     def test_GET_FILEURL_uri_missing(self):
1286         d = self.GET(self.public_url + "/foo/missing?t=uri")
1287         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1288         return d
1289
1290     def test_GET_DIRECTORY_html(self):
1291         d = self.GET(self.public_url + "/foo", followRedirect=True)
1292         def _check(res):
1293             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1294             # These are radio buttons that allow a user to toggle
1295             # whether a particular mutable file is SDMF or MDMF.
1296             self.failUnlessIn("mutable-type-mdmf", res)
1297             self.failUnlessIn("mutable-type-sdmf", res)
1298             # Similarly, these toggle whether a particular directory
1299             # should be MDMF or SDMF.
1300             self.failUnlessIn("mutable-directory-mdmf", res)
1301             self.failUnlessIn("mutable-directory-sdmf", res)
1302             self.failUnlessIn("quux", res)
1303         d.addCallback(_check)
1304         return d
1305
1306     def test_GET_root_html(self):
1307         # make sure that we have the option to upload an unlinked
1308         # mutable file in SDMF and MDMF formats.
1309         d = self.GET("/")
1310         def _got_html(html):
1311             # These are radio buttons that allow the user to toggle
1312             # whether a particular mutable file is MDMF or SDMF.
1313             self.failUnlessIn("mutable-type-mdmf", html)
1314             self.failUnlessIn("mutable-type-sdmf", html)
1315             # We should also have the ability to create a mutable directory.
1316             self.failUnlessIn("mkdir", html)
1317             # ...and we should have the ability to say whether that's an
1318             # MDMF or SDMF directory
1319             self.failUnlessIn("mutable-directory-mdmf", html)
1320             self.failUnlessIn("mutable-directory-sdmf", html)
1321         d.addCallback(_got_html)
1322         return d
1323
1324     def test_mutable_type_defaults(self):
1325         # The checked="checked" attribute of the inputs corresponding to
1326         # the mutable-type parameter should change as expected with the
1327         # value configured in tahoe.cfg.
1328         #
1329         # By default, the value configured with the client is
1330         # SDMF_VERSION, so that should be checked.
1331         assert self.s.mutable_file_default == SDMF_VERSION
1332
1333         d = self.GET("/")
1334         def _got_html(html, value):
1335             i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1336             self.failUnlessIn(i % value, html)
1337         d.addCallback(_got_html, "sdmf")
1338         d.addCallback(lambda ignored:
1339             self.GET(self.public_url + "/foo", followRedirect=True))
1340         d.addCallback(_got_html, "sdmf")
1341         # Now switch the configuration value to MDMF. The MDMF radio
1342         # buttons should now be checked on these pages.
1343         def _swap_values(ignored):
1344             self.s.mutable_file_default = MDMF_VERSION
1345         d.addCallback(_swap_values)
1346         d.addCallback(lambda ignored: self.GET("/"))
1347         d.addCallback(_got_html, "mdmf")
1348         d.addCallback(lambda ignored:
1349             self.GET(self.public_url + "/foo", followRedirect=True))
1350         d.addCallback(_got_html, "mdmf")
1351         return d
1352
1353     def test_GET_DIRURL(self):
1354         # the addSlash means we get a redirect here
1355         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1356         ROOT = "../../.."
1357         d = self.GET(self.public_url + "/foo", followRedirect=True)
1358         def _check(res):
1359             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1360                             in res, res)
1361             # the FILE reference points to a URI, but it should end in bar.txt
1362             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1363                        (ROOT, urllib.quote(self._bar_txt_uri)))
1364             get_bar = "".join([r'<td>FILE</td>',
1365                                r'\s+<td>',
1366                                r'<a href="%s">bar.txt</a>' % bar_url,
1367                                r'</td>',
1368                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1369                                ])
1370             self.failUnless(re.search(get_bar, res), res)
1371             for label in ['unlink', 'rename']:
1372                 for line in res.split("\n"):
1373                     # find the line that contains the relevant button for bar.txt
1374                     if ("form action" in line and
1375                         ('value="%s"' % (label,)) in line and
1376                         'value="bar.txt"' in line):
1377                         # the form target should use a relative URL
1378                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1379                         self.failUnlessIn('action="%s"' % foo_url, line)
1380                         # and the when_done= should too
1381                         #done_url = urllib.quote(???)
1382                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1383
1384                         # 'unlink' needs to use POST because it directly has a side effect
1385                         if label == 'unlink':
1386                             self.failUnlessIn('method="post"', line)
1387                         break
1388                 else:
1389                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1390
1391             # the DIR reference just points to a URI
1392             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1393             get_sub = ((r'<td>DIR</td>')
1394                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1395             self.failUnless(re.search(get_sub, res), res)
1396         d.addCallback(_check)
1397
1398         # look at a readonly directory
1399         d.addCallback(lambda res:
1400                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1401         def _check2(res):
1402             self.failUnless("(read-only)" in res, res)
1403             self.failIf("Upload a file" in res, res)
1404         d.addCallback(_check2)
1405
1406         # and at a directory that contains a readonly directory
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url, followRedirect=True))
1409         def _check3(res):
1410             self.failUnless(re.search('<td>DIR-RO</td>'
1411                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1412         d.addCallback(_check3)
1413
1414         # and an empty directory
1415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1416         def _check4(res):
1417             self.failUnless("directory is empty" in res, res)
1418             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1419             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1420         d.addCallback(_check4)
1421
1422         # and at a literal directory
1423         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1424         d.addCallback(lambda res:
1425                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1426         def _check5(res):
1427             self.failUnless('(immutable)' in res, res)
1428             self.failUnless(re.search('<td>FILE</td>'
1429                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1430         d.addCallback(_check5)
1431         return d
1432
1433     def test_GET_DIRURL_badtype(self):
1434         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1435                                  400, "Bad Request",
1436                                  "bad t=bogus",
1437                                  self.GET,
1438                                  self.public_url + "/foo?t=bogus")
1439         return d
1440
1441     def test_GET_DIRURL_json(self):
1442         d = self.GET(self.public_url + "/foo?t=json")
1443         d.addCallback(self.failUnlessIsFooJSON)
1444         return d
1445
1446     def test_GET_DIRURL_json_format(self):
1447         d = self.PUT(self.public_url + \
1448                      "/foo/sdmf.txt?format=sdmf",
1449                      self.NEWFILE_CONTENTS * 300000)
1450         d.addCallback(lambda ignored:
1451             self.PUT(self.public_url + \
1452                      "/foo/mdmf.txt?format=mdmf",
1453                      self.NEWFILE_CONTENTS * 300000))
1454         # Now we have an MDMF and SDMF file in the directory. If we GET
1455         # its JSON, we should see their encodings.
1456         d.addCallback(lambda ignored:
1457             self.GET(self.public_url + "/foo?t=json"))
1458         def _got_json(json):
1459             data = simplejson.loads(json)
1460             assert data[0] == "dirnode"
1461
1462             data = data[1]
1463             kids = data['children']
1464
1465             mdmf_data = kids['mdmf.txt'][1]
1466             self.failUnlessIn("format", mdmf_data)
1467             self.failUnlessEqual(mdmf_data["format"], "mdmf")
1468
1469             sdmf_data = kids['sdmf.txt'][1]
1470             self.failUnlessIn("format", sdmf_data)
1471             self.failUnlessEqual(sdmf_data["format"], "sdmf")
1472         d.addCallback(_got_json)
1473         return d
1474
1475
1476     def test_POST_DIRURL_manifest_no_ophandle(self):
1477         d = self.shouldFail2(error.Error,
1478                              "test_POST_DIRURL_manifest_no_ophandle",
1479                              "400 Bad Request",
1480                              "slow operation requires ophandle=",
1481                              self.POST, self.public_url, t="start-manifest")
1482         return d
1483
1484     def test_POST_DIRURL_manifest(self):
1485         d = defer.succeed(None)
1486         def getman(ignored, output):
1487             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1488                           followRedirect=True)
1489             d.addCallback(self.wait_for_operation, "125")
1490             d.addCallback(self.get_operation_results, "125", output)
1491             return d
1492         d.addCallback(getman, None)
1493         def _got_html(manifest):
1494             self.failUnless("Manifest of SI=" in manifest)
1495             self.failUnless("<td>sub</td>" in manifest)
1496             self.failUnless(self._sub_uri in manifest)
1497             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1498         d.addCallback(_got_html)
1499
1500         # both t=status and unadorned GET should be identical
1501         d.addCallback(lambda res: self.GET("/operations/125"))
1502         d.addCallback(_got_html)
1503
1504         d.addCallback(getman, "html")
1505         d.addCallback(_got_html)
1506         d.addCallback(getman, "text")
1507         def _got_text(manifest):
1508             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1509             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1510         d.addCallback(_got_text)
1511         d.addCallback(getman, "JSON")
1512         def _got_json(res):
1513             data = res["manifest"]
1514             got = {}
1515             for (path_list, cap) in data:
1516                 got[tuple(path_list)] = cap
1517             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1518             self.failUnless((u"sub",u"baz.txt") in got)
1519             self.failUnless("finished" in res)
1520             self.failUnless("origin" in res)
1521             self.failUnless("storage-index" in res)
1522             self.failUnless("verifycaps" in res)
1523             self.failUnless("stats" in res)
1524         d.addCallback(_got_json)
1525         return d
1526
1527     def test_POST_DIRURL_deepsize_no_ophandle(self):
1528         d = self.shouldFail2(error.Error,
1529                              "test_POST_DIRURL_deepsize_no_ophandle",
1530                              "400 Bad Request",
1531                              "slow operation requires ophandle=",
1532                              self.POST, self.public_url, t="start-deep-size")
1533         return d
1534
1535     def test_POST_DIRURL_deepsize(self):
1536         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1537                       followRedirect=True)
1538         d.addCallback(self.wait_for_operation, "126")
1539         d.addCallback(self.get_operation_results, "126", "json")
1540         def _got_json(data):
1541             self.failUnlessReallyEqual(data["finished"], True)
1542             size = data["size"]
1543             self.failUnless(size > 1000)
1544         d.addCallback(_got_json)
1545         d.addCallback(self.get_operation_results, "126", "text")
1546         def _got_text(res):
1547             mo = re.search(r'^size: (\d+)$', res, re.M)
1548             self.failUnless(mo, res)
1549             size = int(mo.group(1))
1550             # with directories, the size varies.
1551             self.failUnless(size > 1000)
1552         d.addCallback(_got_text)
1553         return d
1554
1555     def test_POST_DIRURL_deepstats_no_ophandle(self):
1556         d = self.shouldFail2(error.Error,
1557                              "test_POST_DIRURL_deepstats_no_ophandle",
1558                              "400 Bad Request",
1559                              "slow operation requires ophandle=",
1560                              self.POST, self.public_url, t="start-deep-stats")
1561         return d
1562
1563     def test_POST_DIRURL_deepstats(self):
1564         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1565                       followRedirect=True)
1566         d.addCallback(self.wait_for_operation, "127")
1567         d.addCallback(self.get_operation_results, "127", "json")
1568         def _got_json(stats):
1569             expected = {"count-immutable-files": 3,
1570                         "count-mutable-files": 2,
1571                         "count-literal-files": 0,
1572                         "count-files": 5,
1573                         "count-directories": 3,
1574                         "size-immutable-files": 57,
1575                         "size-literal-files": 0,
1576                         #"size-directories": 1912, # varies
1577                         #"largest-directory": 1590,
1578                         "largest-directory-children": 7,
1579                         "largest-immutable-file": 19,
1580                         }
1581             for k,v in expected.iteritems():
1582                 self.failUnlessReallyEqual(stats[k], v,
1583                                            "stats[%s] was %s, not %s" %
1584                                            (k, stats[k], v))
1585             self.failUnlessReallyEqual(stats["size-files-histogram"],
1586                                        [ [11, 31, 3] ])
1587         d.addCallback(_got_json)
1588         return d
1589
1590     def test_POST_DIRURL_stream_manifest(self):
1591         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1592         def _check(res):
1593             self.failUnless(res.endswith("\n"))
1594             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1595             self.failUnlessReallyEqual(len(units), 9)
1596             self.failUnlessEqual(units[-1]["type"], "stats")
1597             first = units[0]
1598             self.failUnlessEqual(first["path"], [])
1599             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1600             self.failUnlessEqual(first["type"], "directory")
1601             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1602             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1603             self.failIfEqual(baz["storage-index"], None)
1604             self.failIfEqual(baz["verifycap"], None)
1605             self.failIfEqual(baz["repaircap"], None)
1606             # XXX: Add quux and baz to this test.
1607             return
1608         d.addCallback(_check)
1609         return d
1610
1611     def test_GET_DIRURL_uri(self):
1612         d = self.GET(self.public_url + "/foo?t=uri")
1613         def _check(res):
1614             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1615         d.addCallback(_check)
1616         return d
1617
1618     def test_GET_DIRURL_readonly_uri(self):
1619         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1620         def _check(res):
1621             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1622         d.addCallback(_check)
1623         return d
1624
1625     def test_PUT_NEWDIRURL(self):
1626         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, [])
1631         return d
1632
1633     def test_PUT_NEWDIRURL_mdmf(self):
1634         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1635         d.addCallback(lambda res:
1636                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(lambda node:
1639             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1640         return d
1641
1642     def test_PUT_NEWDIRURL_sdmf(self):
1643         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1644                      "")
1645         d.addCallback(lambda res:
1646                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648         d.addCallback(lambda node:
1649             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1650         return d
1651
1652     def test_PUT_NEWDIRURL_bad_format(self):
1653         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1654                                     400, "Bad Request", "Unknown format: foo",
1655                                     self.PUT, self.public_url +
1656                                     "/foo/newdir=?t=mkdir&format=foo", "")
1657
1658     def test_POST_NEWDIRURL(self):
1659         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1660         d.addCallback(lambda res:
1661                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1662         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1663         d.addCallback(self.failUnlessNodeKeysAre, [])
1664         return d
1665
1666     def test_POST_NEWDIRURL_mdmf(self):
1667         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1668         d.addCallback(lambda res:
1669                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1670         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1671         d.addCallback(lambda node:
1672             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1673         return d
1674
1675     def test_POST_NEWDIRURL_sdmf(self):
1676         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1677         d.addCallback(lambda res:
1678             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1679         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680         d.addCallback(lambda node:
1681             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1682         return d
1683
1684     def test_POST_NEWDIRURL_bad_format(self):
1685         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1686                                     400, "Bad Request", "Unknown format: foo",
1687                                     self.POST2, self.public_url + \
1688                                     "/foo/newdir?t=mkdir&format=foo", "")
1689
1690     def test_POST_NEWDIRURL_emptyname(self):
1691         # an empty pathname component (i.e. a double-slash) is disallowed
1692         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1693                              "400 Bad Request",
1694                              "The webapi does not allow empty pathname components, i.e. a double slash",
1695                              self.POST, self.public_url + "//?t=mkdir")
1696         return d
1697
1698     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1699         (newkids, caps) = self._create_initial_children()
1700         query = "/foo/newdir?t=mkdir-with-children"
1701         if version == MDMF_VERSION:
1702             query += "&format=mdmf"
1703         elif version == SDMF_VERSION:
1704             query += "&format=sdmf"
1705         else:
1706             version = SDMF_VERSION # for later
1707         d = self.POST2(self.public_url + query,
1708                        simplejson.dumps(newkids))
1709         def _check(uri):
1710             n = self.s.create_node_from_uri(uri.strip())
1711             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1712             self.failUnlessEqual(n._node.get_version(), version)
1713             d2.addCallback(lambda ign:
1714                            self.failUnlessROChildURIIs(n, u"child-imm",
1715                                                        caps['filecap1']))
1716             d2.addCallback(lambda ign:
1717                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1718                                                        caps['filecap2']))
1719             d2.addCallback(lambda ign:
1720                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1721                                                        caps['filecap3']))
1722             d2.addCallback(lambda ign:
1723                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1724                                                        caps['unknown_rocap']))
1725             d2.addCallback(lambda ign:
1726                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1727                                                        caps['unknown_rwcap']))
1728             d2.addCallback(lambda ign:
1729                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1730                                                        caps['unknown_immcap']))
1731             d2.addCallback(lambda ign:
1732                            self.failUnlessRWChildURIIs(n, u"dirchild",
1733                                                        caps['dircap']))
1734             d2.addCallback(lambda ign:
1735                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1736                                                        caps['litdircap']))
1737             d2.addCallback(lambda ign:
1738                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1739                                                        caps['emptydircap']))
1740             return d2
1741         d.addCallback(_check)
1742         d.addCallback(lambda res:
1743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1748         return d
1749
1750     def test_POST_NEWDIRURL_initial_children(self):
1751         return self._do_POST_NEWDIRURL_initial_children_test()
1752
1753     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1754         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1755
1756     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1757         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1758
1759     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1760         (newkids, caps) = self._create_initial_children()
1761         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1762                                     400, "Bad Request", "Unknown format: foo",
1763                                     self.POST2, self.public_url + \
1764                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1765                                     simplejson.dumps(newkids))
1766
1767     def test_POST_NEWDIRURL_immutable(self):
1768         (newkids, caps) = self._create_immutable_children()
1769         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1770                        simplejson.dumps(newkids))
1771         def _check(uri):
1772             n = self.s.create_node_from_uri(uri.strip())
1773             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1774             d2.addCallback(lambda ign:
1775                            self.failUnlessROChildURIIs(n, u"child-imm",
1776                                                        caps['filecap1']))
1777             d2.addCallback(lambda ign:
1778                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1779                                                        caps['unknown_immcap']))
1780             d2.addCallback(lambda ign:
1781                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1782                                                        caps['immdircap']))
1783             d2.addCallback(lambda ign:
1784                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1785                                                        caps['litdircap']))
1786             d2.addCallback(lambda ign:
1787                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1788                                                        caps['emptydircap']))
1789             return d2
1790         d.addCallback(_check)
1791         d.addCallback(lambda res:
1792                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1795         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1796         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1800         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1802         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1805         d.addErrback(self.explain_web_error)
1806         return d
1807
1808     def test_POST_NEWDIRURL_immutable_bad(self):
1809         (newkids, caps) = self._create_initial_children()
1810         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1811                              "400 Bad Request",
1812                              "needed to be immutable but was not",
1813                              self.POST2,
1814                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1815                              simplejson.dumps(newkids))
1816         return d
1817
1818     def test_PUT_NEWDIRURL_exists(self):
1819         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1822         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1823         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1824         return d
1825
1826     def test_PUT_NEWDIRURL_blocked(self):
1827         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1828                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1829                              self.PUT,
1830                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1833         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1834         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mkdir_p(self):
1838         d = defer.succeed(None)
1839         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1840         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1841         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1842         def mkdir_p(mkpnode):
1843             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1844             d = self.POST(url)
1845             def made_subsub(ssuri):
1846                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1847                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1848                 d = self.POST(url)
1849                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1850                 return d
1851             d.addCallback(made_subsub)
1852             return d
1853         d.addCallback(mkdir_p)
1854         return d
1855
1856     def test_PUT_NEWDIRURL_mkdirs(self):
1857         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1858         d.addCallback(lambda res:
1859                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1860         d.addCallback(lambda res:
1861                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1862         d.addCallback(lambda res:
1863                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1864         d.addCallback(self.failUnlessNodeKeysAre, [])
1865         return d
1866
1867     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1868         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1869         d.addCallback(lambda ignored:
1870             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1871         d.addCallback(lambda ignored:
1872             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1873         d.addCallback(lambda ignored:
1874             self._foo_node.get_child_at_path(u"subdir"))
1875         def _got_subdir(subdir):
1876             # XXX: What we want?
1877             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1878             self.failUnlessNodeHasChild(subdir, u"newdir")
1879             return subdir.get_child_at_path(u"newdir")
1880         d.addCallback(_got_subdir)
1881         d.addCallback(lambda newdir:
1882             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1883         return d
1884
1885     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1886         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1887         d.addCallback(lambda ignored:
1888             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1889         d.addCallback(lambda ignored:
1890             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1891         d.addCallback(lambda ignored:
1892             self._foo_node.get_child_at_path(u"subdir"))
1893         def _got_subdir(subdir):
1894             # XXX: What we want?
1895             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1896             self.failUnlessNodeHasChild(subdir, u"newdir")
1897             return subdir.get_child_at_path(u"newdir")
1898         d.addCallback(_got_subdir)
1899         d.addCallback(lambda newdir:
1900             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1901         return d
1902
1903     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1904         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1905                                     400, "Bad Request", "Unknown format: foo",
1906                                     self.PUT, self.public_url + \
1907                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1908                                     "")
1909
1910     def test_DELETE_DIRURL(self):
1911         d = self.DELETE(self.public_url + "/foo")
1912         d.addCallback(lambda res:
1913                       self.failIfNodeHasChild(self.public_root, u"foo"))
1914         return d
1915
1916     def test_DELETE_DIRURL_missing(self):
1917         d = self.DELETE(self.public_url + "/foo/missing")
1918         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1919         d.addCallback(lambda res:
1920                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1921         return d
1922
1923     def test_DELETE_DIRURL_missing2(self):
1924         d = self.DELETE(self.public_url + "/missing")
1925         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1926         return d
1927
1928     def dump_root(self):
1929         print "NODEWALK"
1930         w = webish.DirnodeWalkerMixin()
1931         def visitor(childpath, childnode, metadata):
1932             print childpath
1933         d = w.walk(self.public_root, visitor)
1934         return d
1935
1936     def failUnlessNodeKeysAre(self, node, expected_keys):
1937         for k in expected_keys:
1938             assert isinstance(k, unicode)
1939         d = node.list()
1940         def _check(children):
1941             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1942         d.addCallback(_check)
1943         return d
1944     def failUnlessNodeHasChild(self, node, name):
1945         assert isinstance(name, unicode)
1946         d = node.list()
1947         def _check(children):
1948             self.failUnless(name in children)
1949         d.addCallback(_check)
1950         return d
1951     def failIfNodeHasChild(self, node, name):
1952         assert isinstance(name, unicode)
1953         d = node.list()
1954         def _check(children):
1955             self.failIf(name in children)
1956         d.addCallback(_check)
1957         return d
1958
1959     def failUnlessChildContentsAre(self, node, name, expected_contents):
1960         assert isinstance(name, unicode)
1961         d = node.get_child_at_path(name)
1962         d.addCallback(lambda node: download_to_data(node))
1963         def _check(contents):
1964             self.failUnlessReallyEqual(contents, expected_contents)
1965         d.addCallback(_check)
1966         return d
1967
1968     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1969         assert isinstance(name, unicode)
1970         d = node.get_child_at_path(name)
1971         d.addCallback(lambda node: node.download_best_version())
1972         def _check(contents):
1973             self.failUnlessReallyEqual(contents, expected_contents)
1974         d.addCallback(_check)
1975         return d
1976
1977     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1978         assert isinstance(name, unicode)
1979         d = node.get_child_at_path(name)
1980         def _check(child):
1981             self.failUnless(child.is_unknown() or not child.is_readonly())
1982             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1983             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1984             expected_ro_uri = self._make_readonly(expected_uri)
1985             if expected_ro_uri:
1986                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1987         d.addCallback(_check)
1988         return d
1989
1990     def failUnlessROChildURIIs(self, node, name, expected_uri):
1991         assert isinstance(name, unicode)
1992         d = node.get_child_at_path(name)
1993         def _check(child):
1994             self.failUnless(child.is_unknown() or child.is_readonly())
1995             self.failUnlessReallyEqual(child.get_write_uri(), None)
1996             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1997             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1998         d.addCallback(_check)
1999         return d
2000
2001     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2002         assert isinstance(name, unicode)
2003         d = node.get_child_at_path(name)
2004         def _check(child):
2005             self.failUnless(child.is_unknown() or not child.is_readonly())
2006             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2007             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2008             expected_ro_uri = self._make_readonly(got_uri)
2009             if expected_ro_uri:
2010                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2011         d.addCallback(_check)
2012         return d
2013
2014     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2015         assert isinstance(name, unicode)
2016         d = node.get_child_at_path(name)
2017         def _check(child):
2018             self.failUnless(child.is_unknown() or child.is_readonly())
2019             self.failUnlessReallyEqual(child.get_write_uri(), None)
2020             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2021             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2022         d.addCallback(_check)
2023         return d
2024
2025     def failUnlessCHKURIHasContents(self, got_uri, contents):
2026         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2027
2028     def test_POST_upload(self):
2029         d = self.POST(self.public_url + "/foo", t="upload",
2030                       file=("new.txt", self.NEWFILE_CONTENTS))
2031         fn = self._foo_node
2032         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2033         d.addCallback(lambda res:
2034                       self.failUnlessChildContentsAre(fn, u"new.txt",
2035                                                       self.NEWFILE_CONTENTS))
2036         return d
2037
2038     def test_POST_upload_unicode(self):
2039         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2040         d = self.POST(self.public_url + "/foo", t="upload",
2041                       file=(filename, self.NEWFILE_CONTENTS))
2042         fn = self._foo_node
2043         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2044         d.addCallback(lambda res:
2045                       self.failUnlessChildContentsAre(fn, filename,
2046                                                       self.NEWFILE_CONTENTS))
2047         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2048         d.addCallback(lambda res: self.GET(target_url))
2049         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2050                                                                   self.NEWFILE_CONTENTS,
2051                                                                   contents))
2052         return d
2053
2054     def test_POST_upload_unicode_named(self):
2055         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2056         d = self.POST(self.public_url + "/foo", t="upload",
2057                       name=filename,
2058                       file=("overridden", self.NEWFILE_CONTENTS))
2059         fn = self._foo_node
2060         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2061         d.addCallback(lambda res:
2062                       self.failUnlessChildContentsAre(fn, filename,
2063                                                       self.NEWFILE_CONTENTS))
2064         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2065         d.addCallback(lambda res: self.GET(target_url))
2066         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2067                                                                   self.NEWFILE_CONTENTS,
2068                                                                   contents))
2069         return d
2070
2071     def test_POST_upload_no_link(self):
2072         d = self.POST("/uri", t="upload",
2073                       file=("new.txt", self.NEWFILE_CONTENTS))
2074         def _check_upload_results(page):
2075             # this should be a page which describes the results of the upload
2076             # that just finished.
2077             self.failUnless("Upload Results:" in page)
2078             self.failUnless("URI:" in page)
2079             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2080             mo = uri_re.search(page)
2081             self.failUnless(mo, page)
2082             new_uri = mo.group(1)
2083             return new_uri
2084         d.addCallback(_check_upload_results)
2085         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2086         return d
2087
2088     def test_POST_upload_no_link_whendone(self):
2089         d = self.POST("/uri", t="upload", when_done="/",
2090                       file=("new.txt", self.NEWFILE_CONTENTS))
2091         d.addBoth(self.shouldRedirect, "/")
2092         return d
2093
2094     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2095         d = defer.maybeDeferred(callable, *args, **kwargs)
2096         def done(res):
2097             if isinstance(res, failure.Failure):
2098                 res.trap(error.PageRedirect)
2099                 statuscode = res.value.status
2100                 target = res.value.location
2101                 return checker(statuscode, target)
2102             self.fail("%s: callable was supposed to redirect, not return '%s'"
2103                       % (which, res))
2104         d.addBoth(done)
2105         return d
2106
2107     def test_POST_upload_no_link_whendone_results(self):
2108         def check(statuscode, target):
2109             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2110             self.failUnless(target.startswith(self.webish_url), target)
2111             return client.getPage(target, method="GET")
2112         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2113                                  check,
2114                                  self.POST, "/uri", t="upload",
2115                                  when_done="/uri/%(uri)s",
2116                                  file=("new.txt", self.NEWFILE_CONTENTS))
2117         d.addCallback(lambda res:
2118                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2119         return d
2120
2121     def test_POST_upload_no_link_mutable(self):
2122         d = self.POST("/uri", t="upload", mutable="true",
2123                       file=("new.txt", self.NEWFILE_CONTENTS))
2124         def _check(filecap):
2125             filecap = filecap.strip()
2126             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2127             self.filecap = filecap
2128             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2129             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2130             n = self.s.create_node_from_uri(filecap)
2131             return n.download_best_version()
2132         d.addCallback(_check)
2133         def _check2(data):
2134             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2135             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2136         d.addCallback(_check2)
2137         def _check3(data):
2138             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2139             return self.GET("/file/%s" % urllib.quote(self.filecap))
2140         d.addCallback(_check3)
2141         def _check4(data):
2142             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2143         d.addCallback(_check4)
2144         return d
2145
2146     def test_POST_upload_no_link_mutable_toobig(self):
2147         # The SDMF size limit is no longer in place, so we should be
2148         # able to upload mutable files that are as large as we want them
2149         # to be.
2150         d = self.POST("/uri", t="upload", mutable="true",
2151                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2152         return d
2153
2154
2155     def test_POST_upload_format_unlinked(self):
2156         def _check_upload_unlinked(ign, format, uri_prefix):
2157             filename = format + ".txt"
2158             d = self.POST("/uri?t=upload&format=" + format,
2159                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2160             def _got_filecap(filecap):
2161                 self.failUnless(filecap.startswith(uri_prefix))
2162                 return self.GET("/uri/%s?t=json" % filecap)
2163             d.addCallback(_got_filecap)
2164             def _got_json(json):
2165                 data = simplejson.loads(json)
2166                 data = data[1]
2167                 self.failUnlessIn("format", data)
2168                 self.failUnlessEqual(data["format"], format)
2169             d.addCallback(_got_json)
2170
2171         d = defer.succeed(None)
2172         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2173         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2174         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2175         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2176         return d
2177
2178     def test_POST_upload_bad_format_unlinked(self):
2179         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2180                                     400, "Bad Request", "Unknown format: foo",
2181                                     self.POST,
2182                                     "/uri?t=upload&format=foo",
2183                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2184
2185     def test_POST_upload_format(self):
2186         def _check_upload(ign, format, uri_prefix, fn=None):
2187             filename = format + ".txt"
2188             d = self.POST(self.public_url +
2189                           "/foo?t=upload&format=" + format,
2190                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2191             def _got_filecap(filecap):
2192                 if fn is not None:
2193                     filenameu = unicode(filename)
2194                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2195                 self.failUnless(filecap.startswith(uri_prefix))
2196                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2197             d.addCallback(_got_filecap)
2198             def _got_json(json):
2199                 data = simplejson.loads(json)
2200                 data = data[1]
2201                 self.failUnlessIn("format", data)
2202                 self.failUnlessEqual(data["format"], format)
2203             d.addCallback(_got_json)
2204
2205         d = defer.succeed(None)
2206         d.addCallback(_check_upload, "chk", "URI:CHK")
2207         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2208         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2209         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2210         return d
2211
2212     def test_POST_upload_bad_format(self):
2213         return self.shouldHTTPError("POST_upload_bad_format",
2214                                     400, "Bad Request", "Unknown format: foo",
2215                                     self.POST, self.public_url + \
2216                                     "/foo?t=upload&format=foo",
2217                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2218
2219     def test_POST_upload_mutable(self):
2220         # this creates a mutable file
2221         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2222                       file=("new.txt", self.NEWFILE_CONTENTS))
2223         fn = self._foo_node
2224         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2225         d.addCallback(lambda res:
2226                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2227                                                              self.NEWFILE_CONTENTS))
2228         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2229         def _got(newnode):
2230             self.failUnless(IMutableFileNode.providedBy(newnode))
2231             self.failUnless(newnode.is_mutable())
2232             self.failIf(newnode.is_readonly())
2233             self._mutable_node = newnode
2234             self._mutable_uri = newnode.get_uri()
2235         d.addCallback(_got)
2236
2237         # now upload it again and make sure that the URI doesn't change
2238         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2239         d.addCallback(lambda res:
2240                       self.POST(self.public_url + "/foo", t="upload",
2241                                 mutable="true",
2242                                 file=("new.txt", NEWER_CONTENTS)))
2243         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2244         d.addCallback(lambda res:
2245                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2246                                                              NEWER_CONTENTS))
2247         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2248         def _got2(newnode):
2249             self.failUnless(IMutableFileNode.providedBy(newnode))
2250             self.failUnless(newnode.is_mutable())
2251             self.failIf(newnode.is_readonly())
2252             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2253         d.addCallback(_got2)
2254
2255         # upload a second time, using PUT instead of POST
2256         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2257         d.addCallback(lambda res:
2258                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2259         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2260         d.addCallback(lambda res:
2261                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2262                                                              NEW2_CONTENTS))
2263
2264         # finally list the directory, since mutable files are displayed
2265         # slightly differently
2266
2267         d.addCallback(lambda res:
2268                       self.GET(self.public_url + "/foo/",
2269                                followRedirect=True))
2270         def _check_page(res):
2271             # TODO: assert more about the contents
2272             self.failUnless("SSK" in res)
2273             return res
2274         d.addCallback(_check_page)
2275
2276         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2277         def _got3(newnode):
2278             self.failUnless(IMutableFileNode.providedBy(newnode))
2279             self.failUnless(newnode.is_mutable())
2280             self.failIf(newnode.is_readonly())
2281             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2282         d.addCallback(_got3)
2283
2284         # look at the JSON form of the enclosing directory
2285         d.addCallback(lambda res:
2286                       self.GET(self.public_url + "/foo/?t=json",
2287                                followRedirect=True))
2288         def _check_page_json(res):
2289             parsed = simplejson.loads(res)
2290             self.failUnlessEqual(parsed[0], "dirnode")
2291             children = dict( [(unicode(name),value)
2292                               for (name,value)
2293                               in parsed[1]["children"].iteritems()] )
2294             self.failUnless(u"new.txt" in children)
2295             new_json = children[u"new.txt"]
2296             self.failUnlessEqual(new_json[0], "filenode")
2297             self.failUnless(new_json[1]["mutable"])
2298             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2299             ro_uri = self._mutable_node.get_readonly().to_string()
2300             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2301         d.addCallback(_check_page_json)
2302
2303         # and the JSON form of the file
2304         d.addCallback(lambda res:
2305                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2306         def _check_file_json(res):
2307             parsed = simplejson.loads(res)
2308             self.failUnlessEqual(parsed[0], "filenode")
2309             self.failUnless(parsed[1]["mutable"])
2310             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2311             ro_uri = self._mutable_node.get_readonly().to_string()
2312             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2313         d.addCallback(_check_file_json)
2314
2315         # and look at t=uri and t=readonly-uri
2316         d.addCallback(lambda res:
2317                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2318         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2319         d.addCallback(lambda res:
2320                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2321         def _check_ro_uri(res):
2322             ro_uri = self._mutable_node.get_readonly().to_string()
2323             self.failUnlessReallyEqual(res, ro_uri)
2324         d.addCallback(_check_ro_uri)
2325
2326         # make sure we can get to it from /uri/URI
2327         d.addCallback(lambda res:
2328                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2329         d.addCallback(lambda res:
2330                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2331
2332         # and that HEAD computes the size correctly
2333         d.addCallback(lambda res:
2334                       self.HEAD(self.public_url + "/foo/new.txt",
2335                                 return_response=True))
2336         def _got_headers((res, status, headers)):
2337             self.failUnlessReallyEqual(res, "")
2338             self.failUnlessReallyEqual(headers["content-length"][0],
2339                                        str(len(NEW2_CONTENTS)))
2340             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2341         d.addCallback(_got_headers)
2342
2343         # make sure that outdated size limits aren't enforced anymore.
2344         d.addCallback(lambda ignored:
2345             self.POST(self.public_url + "/foo", t="upload",
2346                       mutable="true",
2347                       file=("new.txt",
2348                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2349         d.addErrback(self.dump_error)
2350         return d
2351
2352     def test_POST_upload_mutable_toobig(self):
2353         # SDMF had a size limti that was removed a while ago. MDMF has
2354         # never had a size limit. Test to make sure that we do not
2355         # encounter errors when trying to upload large mutable files,
2356         # since there should be no coded prohibitions regarding large
2357         # mutable files.
2358         d = self.POST(self.public_url + "/foo",
2359                       t="upload", mutable="true",
2360                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2361         return d
2362
2363     def dump_error(self, f):
2364         # if the web server returns an error code (like 400 Bad Request),
2365         # web.client.getPage puts the HTTP response body into the .response
2366         # attribute of the exception object that it gives back. It does not
2367         # appear in the Failure's repr(), so the ERROR that trial displays
2368         # will be rather terse and unhelpful. addErrback this method to the
2369         # end of your chain to get more information out of these errors.
2370         if f.check(error.Error):
2371             print "web.error.Error:"
2372             print f
2373             print f.value.response
2374         return f
2375
2376     def test_POST_upload_replace(self):
2377         d = self.POST(self.public_url + "/foo", t="upload",
2378                       file=("bar.txt", self.NEWFILE_CONTENTS))
2379         fn = self._foo_node
2380         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2381         d.addCallback(lambda res:
2382                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2383                                                       self.NEWFILE_CONTENTS))
2384         return d
2385
2386     def test_POST_upload_no_replace_ok(self):
2387         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2388                       file=("new.txt", self.NEWFILE_CONTENTS))
2389         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2390         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2391                                                              self.NEWFILE_CONTENTS))
2392         return d
2393
2394     def test_POST_upload_no_replace_queryarg(self):
2395         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2396                       file=("bar.txt", self.NEWFILE_CONTENTS))
2397         d.addBoth(self.shouldFail, error.Error,
2398                   "POST_upload_no_replace_queryarg",
2399                   "409 Conflict",
2400                   "There was already a child by that name, and you asked me "
2401                   "to not replace it")
2402         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2403         d.addCallback(self.failUnlessIsBarDotTxt)
2404         return d
2405
2406     def test_POST_upload_no_replace_field(self):
2407         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2408                       file=("bar.txt", self.NEWFILE_CONTENTS))
2409         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2410                   "409 Conflict",
2411                   "There was already a child by that name, and you asked me "
2412                   "to not replace it")
2413         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2414         d.addCallback(self.failUnlessIsBarDotTxt)
2415         return d
2416
2417     def test_POST_upload_whendone(self):
2418         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2419                       file=("new.txt", self.NEWFILE_CONTENTS))
2420         d.addBoth(self.shouldRedirect, "/THERE")
2421         fn = self._foo_node
2422         d.addCallback(lambda res:
2423                       self.failUnlessChildContentsAre(fn, u"new.txt",
2424                                                       self.NEWFILE_CONTENTS))
2425         return d
2426
2427     def test_POST_upload_named(self):
2428         fn = self._foo_node
2429         d = self.POST(self.public_url + "/foo", t="upload",
2430                       name="new.txt", file=self.NEWFILE_CONTENTS)
2431         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2432         d.addCallback(lambda res:
2433                       self.failUnlessChildContentsAre(fn, u"new.txt",
2434                                                       self.NEWFILE_CONTENTS))
2435         return d
2436
2437     def test_POST_upload_named_badfilename(self):
2438         d = self.POST(self.public_url + "/foo", t="upload",
2439                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2440         d.addBoth(self.shouldFail, error.Error,
2441                   "test_POST_upload_named_badfilename",
2442                   "400 Bad Request",
2443                   "name= may not contain a slash",
2444                   )
2445         # make sure that nothing was added
2446         d.addCallback(lambda res:
2447                       self.failUnlessNodeKeysAre(self._foo_node,
2448                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2449                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2450                                                   u"sub"]))
2451         return d
2452
2453     def test_POST_FILEURL_check(self):
2454         bar_url = self.public_url + "/foo/bar.txt"
2455         d = self.POST(bar_url, t="check")
2456         def _check(res):
2457             self.failUnless("Healthy :" in res)
2458         d.addCallback(_check)
2459         redir_url = "http://allmydata.org/TARGET"
2460         def _check2(statuscode, target):
2461             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2462             self.failUnlessReallyEqual(target, redir_url)
2463         d.addCallback(lambda res:
2464                       self.shouldRedirect2("test_POST_FILEURL_check",
2465                                            _check2,
2466                                            self.POST, bar_url,
2467                                            t="check",
2468                                            when_done=redir_url))
2469         d.addCallback(lambda res:
2470                       self.POST(bar_url, t="check", return_to=redir_url))
2471         def _check3(res):
2472             self.failUnless("Healthy :" in res)
2473             self.failUnless("Return to file" in res)
2474             self.failUnless(redir_url in res)
2475         d.addCallback(_check3)
2476
2477         d.addCallback(lambda res:
2478                       self.POST(bar_url, t="check", output="JSON"))
2479         def _check_json(res):
2480             data = simplejson.loads(res)
2481             self.failUnless("storage-index" in data)
2482             self.failUnless(data["results"]["healthy"])
2483         d.addCallback(_check_json)
2484
2485         return d
2486
2487     def test_POST_FILEURL_check_and_repair(self):
2488         bar_url = self.public_url + "/foo/bar.txt"
2489         d = self.POST(bar_url, t="check", repair="true")
2490         def _check(res):
2491             self.failUnless("Healthy :" in res)
2492         d.addCallback(_check)
2493         redir_url = "http://allmydata.org/TARGET"
2494         def _check2(statuscode, target):
2495             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2496             self.failUnlessReallyEqual(target, redir_url)
2497         d.addCallback(lambda res:
2498                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2499                                            _check2,
2500                                            self.POST, bar_url,
2501                                            t="check", repair="true",
2502                                            when_done=redir_url))
2503         d.addCallback(lambda res:
2504                       self.POST(bar_url, t="check", return_to=redir_url))
2505         def _check3(res):
2506             self.failUnless("Healthy :" in res)
2507             self.failUnless("Return to file" in res)
2508             self.failUnless(redir_url in res)
2509         d.addCallback(_check3)
2510         return d
2511
2512     def test_POST_DIRURL_check(self):
2513         foo_url = self.public_url + "/foo/"
2514         d = self.POST(foo_url, t="check")
2515         def _check(res):
2516             self.failUnless("Healthy :" in res, res)
2517         d.addCallback(_check)
2518         redir_url = "http://allmydata.org/TARGET"
2519         def _check2(statuscode, target):
2520             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2521             self.failUnlessReallyEqual(target, redir_url)
2522         d.addCallback(lambda res:
2523                       self.shouldRedirect2("test_POST_DIRURL_check",
2524                                            _check2,
2525                                            self.POST, foo_url,
2526                                            t="check",
2527                                            when_done=redir_url))
2528         d.addCallback(lambda res:
2529                       self.POST(foo_url, t="check", return_to=redir_url))
2530         def _check3(res):
2531             self.failUnless("Healthy :" in res, res)
2532             self.failUnless("Return to file/directory" in res)
2533             self.failUnless(redir_url in res)
2534         d.addCallback(_check3)
2535
2536         d.addCallback(lambda res:
2537                       self.POST(foo_url, t="check", output="JSON"))
2538         def _check_json(res):
2539             data = simplejson.loads(res)
2540             self.failUnless("storage-index" in data)
2541             self.failUnless(data["results"]["healthy"])
2542         d.addCallback(_check_json)
2543
2544         return d
2545
2546     def test_POST_DIRURL_check_and_repair(self):
2547         foo_url = self.public_url + "/foo/"
2548         d = self.POST(foo_url, t="check", repair="true")
2549         def _check(res):
2550             self.failUnless("Healthy :" in res, res)
2551         d.addCallback(_check)
2552         redir_url = "http://allmydata.org/TARGET"
2553         def _check2(statuscode, target):
2554             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2555             self.failUnlessReallyEqual(target, redir_url)
2556         d.addCallback(lambda res:
2557                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2558                                            _check2,
2559                                            self.POST, foo_url,
2560                                            t="check", repair="true",
2561                                            when_done=redir_url))
2562         d.addCallback(lambda res:
2563                       self.POST(foo_url, t="check", return_to=redir_url))
2564         def _check3(res):
2565             self.failUnless("Healthy :" in res)
2566             self.failUnless("Return to file/directory" in res)
2567             self.failUnless(redir_url in res)
2568         d.addCallback(_check3)
2569         return d
2570
2571     def test_POST_FILEURL_mdmf_check(self):
2572         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2573         d = self.POST(quux_url, t="check")
2574         def _check(res):
2575             self.failUnlessIn("Healthy", res)
2576         d.addCallback(_check)
2577         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2578         d.addCallback(lambda ignored:
2579             self.POST(quux_extension_url, t="check"))
2580         d.addCallback(_check)
2581         return d
2582
2583     def test_POST_FILEURL_mdmf_check_and_repair(self):
2584         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2585         d = self.POST(quux_url, t="check", repair="true")
2586         def _check(res):
2587             self.failUnlessIn("Healthy", res)
2588         d.addCallback(_check)
2589         quux_extension_url = "/uri/%s" %\
2590             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2591         d.addCallback(lambda ignored:
2592             self.POST(quux_extension_url, t="check", repair="true"))
2593         d.addCallback(_check)
2594         return d
2595
2596     def wait_for_operation(self, ignored, ophandle):
2597         url = "/operations/" + ophandle
2598         url += "?t=status&output=JSON"
2599         d = self.GET(url)
2600         def _got(res):
2601             data = simplejson.loads(res)
2602             if not data["finished"]:
2603                 d = self.stall(delay=1.0)
2604                 d.addCallback(self.wait_for_operation, ophandle)
2605                 return d
2606             return data
2607         d.addCallback(_got)
2608         return d
2609
2610     def get_operation_results(self, ignored, ophandle, output=None):
2611         url = "/operations/" + ophandle
2612         url += "?t=status"
2613         if output:
2614             url += "&output=" + output
2615         d = self.GET(url)
2616         def _got(res):
2617             if output and output.lower() == "json":
2618                 return simplejson.loads(res)
2619             return res
2620         d.addCallback(_got)
2621         return d
2622
2623     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2624         d = self.shouldFail2(error.Error,
2625                              "test_POST_DIRURL_deepcheck_no_ophandle",
2626                              "400 Bad Request",
2627                              "slow operation requires ophandle=",
2628                              self.POST, self.public_url, t="start-deep-check")
2629         return d
2630
2631     def test_POST_DIRURL_deepcheck(self):
2632         def _check_redirect(statuscode, target):
2633             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2634             self.failUnless(target.endswith("/operations/123"))
2635         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2636                                  self.POST, self.public_url,
2637                                  t="start-deep-check", ophandle="123")
2638         d.addCallback(self.wait_for_operation, "123")
2639         def _check_json(data):
2640             self.failUnlessReallyEqual(data["finished"], True)
2641             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2642             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2643         d.addCallback(_check_json)
2644         d.addCallback(self.get_operation_results, "123", "html")
2645         def _check_html(res):
2646             self.failUnless("Objects Checked: <span>10</span>" in res)
2647             self.failUnless("Objects Healthy: <span>10</span>" in res)
2648         d.addCallback(_check_html)
2649
2650         d.addCallback(lambda res:
2651                       self.GET("/operations/123/"))
2652         d.addCallback(_check_html) # should be the same as without the slash
2653
2654         d.addCallback(lambda res:
2655                       self.shouldFail2(error.Error, "one", "404 Not Found",
2656                                        "No detailed results for SI bogus",
2657                                        self.GET, "/operations/123/bogus"))
2658
2659         foo_si = self._foo_node.get_storage_index()
2660         foo_si_s = base32.b2a(foo_si)
2661         d.addCallback(lambda res:
2662                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2663         def _check_foo_json(res):
2664             data = simplejson.loads(res)
2665             self.failUnlessEqual(data["storage-index"], foo_si_s)
2666             self.failUnless(data["results"]["healthy"])
2667         d.addCallback(_check_foo_json)
2668         return d
2669
2670     def test_POST_DIRURL_deepcheck_and_repair(self):
2671         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2672                       ophandle="124", output="json", followRedirect=True)
2673         d.addCallback(self.wait_for_operation, "124")
2674         def _check_json(data):
2675             self.failUnlessReallyEqual(data["finished"], True)
2676             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2677             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2678             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2679             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2680             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2681             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2682             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2683             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2684             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2685             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2686         d.addCallback(_check_json)
2687         d.addCallback(self.get_operation_results, "124", "html")
2688         def _check_html(res):
2689             self.failUnless("Objects Checked: <span>10</span>" in res)
2690
2691             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2692             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2693             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2694
2695             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2696             self.failUnless("Repairs Successful: <span>0</span>" in res)
2697             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2698
2699             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2700             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2701             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2702         d.addCallback(_check_html)
2703         return d
2704
2705     def test_POST_FILEURL_bad_t(self):
2706         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2707                              "POST to file: bad t=bogus",
2708                              self.POST, self.public_url + "/foo/bar.txt",
2709                              t="bogus")
2710         return d
2711
2712     def test_POST_mkdir(self): # return value?
2713         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(self.failUnlessNodeKeysAre, [])
2716         return d
2717
2718     def test_POST_mkdir_mdmf(self):
2719         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2720         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2721         d.addCallback(lambda node:
2722             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2723         return d
2724
2725     def test_POST_mkdir_sdmf(self):
2726         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2727         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2728         d.addCallback(lambda node:
2729             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2730         return d
2731
2732     def test_POST_mkdir_bad_format(self):
2733         return self.shouldHTTPError("POST_mkdir_bad_format",
2734                                     400, "Bad Request", "Unknown format: foo",
2735                                     self.POST, self.public_url +
2736                                     "/foo?t=mkdir&name=newdir&format=foo")
2737
2738     def test_POST_mkdir_initial_children(self):
2739         (newkids, caps) = self._create_initial_children()
2740         d = self.POST2(self.public_url +
2741                        "/foo?t=mkdir-with-children&name=newdir",
2742                        simplejson.dumps(newkids))
2743         d.addCallback(lambda res:
2744                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2745         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2746         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2747         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2748         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2749         return d
2750
2751     def test_POST_mkdir_initial_children_mdmf(self):
2752         (newkids, caps) = self._create_initial_children()
2753         d = self.POST2(self.public_url +
2754                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2755                        simplejson.dumps(newkids))
2756         d.addCallback(lambda res:
2757                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2758         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2759         d.addCallback(lambda node:
2760             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2762         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2763                        caps['filecap1'])
2764         return d
2765
2766     # XXX: Duplication.
2767     def test_POST_mkdir_initial_children_sdmf(self):
2768         (newkids, caps) = self._create_initial_children()
2769         d = self.POST2(self.public_url +
2770                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2771                        simplejson.dumps(newkids))
2772         d.addCallback(lambda res:
2773                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2774         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2775         d.addCallback(lambda node:
2776             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2777         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2778         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2779                        caps['filecap1'])
2780         return d
2781
2782     def test_POST_mkdir_initial_children_bad_format(self):
2783         (newkids, caps) = self._create_initial_children()
2784         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2785                                     400, "Bad Request", "Unknown format: foo",
2786                                     self.POST, self.public_url + \
2787                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2788                                     simplejson.dumps(newkids))
2789
2790     def test_POST_mkdir_immutable(self):
2791         (newkids, caps) = self._create_immutable_children()
2792         d = self.POST2(self.public_url +
2793                        "/foo?t=mkdir-immutable&name=newdir",
2794                        simplejson.dumps(newkids))
2795         d.addCallback(lambda res:
2796                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2798         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2800         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2802         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2805         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2806         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2807         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2808         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2809         return d
2810
2811     def test_POST_mkdir_immutable_bad(self):
2812         (newkids, caps) = self._create_initial_children()
2813         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2814                              "400 Bad Request",
2815                              "needed to be immutable but was not",
2816                              self.POST2,
2817                              self.public_url +
2818                              "/foo?t=mkdir-immutable&name=newdir",
2819                              simplejson.dumps(newkids))
2820         return d
2821
2822     def test_POST_mkdir_2(self):
2823         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2824         d.addCallback(lambda res:
2825                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2826         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2827         d.addCallback(self.failUnlessNodeKeysAre, [])
2828         return d
2829
2830     def test_POST_mkdirs_2(self):
2831         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2832         d.addCallback(lambda res:
2833                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2834         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2835         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2836         d.addCallback(self.failUnlessNodeKeysAre, [])
2837         return d
2838
2839     def test_POST_mkdir_no_parentdir_noredirect(self):
2840         d = self.POST("/uri?t=mkdir")
2841         def _after_mkdir(res):
2842             uri.DirectoryURI.init_from_string(res)
2843         d.addCallback(_after_mkdir)
2844         return d
2845
2846     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2847         d = self.POST("/uri?t=mkdir&format=mdmf")
2848         def _after_mkdir(res):
2849             u = uri.from_string(res)
2850             # Check that this is an MDMF writecap
2851             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2852         d.addCallback(_after_mkdir)
2853         return d
2854
2855     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2856         d = self.POST("/uri?t=mkdir&format=sdmf")
2857         def _after_mkdir(res):
2858             u = uri.from_string(res)
2859             self.failUnlessIsInstance(u, uri.DirectoryURI)
2860         d.addCallback(_after_mkdir)
2861         return d
2862
2863     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2864         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2865                                     400, "Bad Request", "Unknown format: foo",
2866                                     self.POST, self.public_url +
2867                                     "/uri?t=mkdir&format=foo")
2868
2869     def test_POST_mkdir_no_parentdir_noredirect2(self):
2870         # make sure form-based arguments (as on the welcome page) still work
2871         d = self.POST("/uri", t="mkdir")
2872         def _after_mkdir(res):
2873             uri.DirectoryURI.init_from_string(res)
2874         d.addCallback(_after_mkdir)
2875         d.addErrback(self.explain_web_error)
2876         return d
2877
2878     def test_POST_mkdir_no_parentdir_redirect(self):
2879         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2880         d.addBoth(self.shouldRedirect, None, statuscode='303')
2881         def _check_target(target):
2882             target = urllib.unquote(target)
2883             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2884         d.addCallback(_check_target)
2885         return d
2886
2887     def test_POST_mkdir_no_parentdir_redirect2(self):
2888         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2889         d.addBoth(self.shouldRedirect, None, statuscode='303')
2890         def _check_target(target):
2891             target = urllib.unquote(target)
2892             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2893         d.addCallback(_check_target)
2894         d.addErrback(self.explain_web_error)
2895         return d
2896
2897     def _make_readonly(self, u):
2898         ro_uri = uri.from_string(u).get_readonly()
2899         if ro_uri is None:
2900             return None
2901         return ro_uri.to_string()
2902
2903     def _create_initial_children(self):
2904         contents, n, filecap1 = self.makefile(12)
2905         md1 = {"metakey1": "metavalue1"}
2906         filecap2 = make_mutable_file_uri()
2907         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2908         filecap3 = node3.get_readonly_uri()
2909         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2910         dircap = DirectoryNode(node4, None, None).get_uri()
2911         mdmfcap = make_mutable_file_uri(mdmf=True)
2912         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2913         emptydircap = "URI:DIR2-LIT:"
2914         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2915                                                       "ro_uri": self._make_readonly(filecap1),
2916                                                       "metadata": md1, }],
2917                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2918                                                       "ro_uri": self._make_readonly(filecap2)}],
2919                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2920                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2921                                                       "ro_uri": unknown_rocap}],
2922                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2923                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2924                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2925                                                       "ro_uri": self._make_readonly(dircap)}],
2926                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2927                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2928                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2929                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2930                    }
2931         return newkids, {'filecap1': filecap1,
2932                          'filecap2': filecap2,
2933                          'filecap3': filecap3,
2934                          'unknown_rwcap': unknown_rwcap,
2935                          'unknown_rocap': unknown_rocap,
2936                          'unknown_immcap': unknown_immcap,
2937                          'dircap': dircap,
2938                          'litdircap': litdircap,
2939                          'emptydircap': emptydircap,
2940                          'mdmfcap': mdmfcap}
2941
2942     def _create_immutable_children(self):
2943         contents, n, filecap1 = self.makefile(12)
2944         md1 = {"metakey1": "metavalue1"}
2945         tnode = create_chk_filenode("immutable directory contents\n"*10)
2946         dnode = DirectoryNode(tnode, None, None)
2947         assert not dnode.is_mutable()
2948         immdircap = dnode.get_uri()
2949         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2950         emptydircap = "URI:DIR2-LIT:"
2951         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2952                                                       "metadata": md1, }],
2953                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2954                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2955                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2956                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2957                    }
2958         return newkids, {'filecap1': filecap1,
2959                          'unknown_immcap': unknown_immcap,
2960                          'immdircap': immdircap,
2961                          'litdircap': litdircap,
2962                          'emptydircap': emptydircap}
2963
2964     def test_POST_mkdir_no_parentdir_initial_children(self):
2965         (newkids, caps) = self._create_initial_children()
2966         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2967         def _after_mkdir(res):
2968             self.failUnless(res.startswith("URI:DIR"), res)
2969             n = self.s.create_node_from_uri(res)
2970             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2971             d2.addCallback(lambda ign:
2972                            self.failUnlessROChildURIIs(n, u"child-imm",
2973                                                        caps['filecap1']))
2974             d2.addCallback(lambda ign:
2975                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2976                                                        caps['filecap2']))
2977             d2.addCallback(lambda ign:
2978                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2979                                                        caps['filecap3']))
2980             d2.addCallback(lambda ign:
2981                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2982                                                        caps['unknown_rwcap']))
2983             d2.addCallback(lambda ign:
2984                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2985                                                        caps['unknown_rocap']))
2986             d2.addCallback(lambda ign:
2987                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2988                                                        caps['unknown_immcap']))
2989             d2.addCallback(lambda ign:
2990                            self.failUnlessRWChildURIIs(n, u"dirchild",
2991                                                        caps['dircap']))
2992             return d2
2993         d.addCallback(_after_mkdir)
2994         return d
2995
2996     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2997         # the regular /uri?t=mkdir operation is specified to ignore its body.
2998         # Only t=mkdir-with-children pays attention to it.
2999         (newkids, caps) = self._create_initial_children()
3000         d = self.shouldHTTPError("POST t=mkdir unexpected children",
3001                                  400, "Bad Request",
3002                                  "t=mkdir does not accept children=, "
3003                                  "try t=mkdir-with-children instead",
3004                                  self.POST2, "/uri?t=mkdir", # without children
3005                                  simplejson.dumps(newkids))
3006         return d
3007
3008     def test_POST_noparent_bad(self):
3009         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
3010                                  "/uri accepts only PUT, PUT?t=mkdir, "
3011                                  "POST?t=upload, and POST?t=mkdir",
3012                                  self.POST, "/uri?t=bogus")
3013         return d
3014
3015     def test_POST_mkdir_no_parentdir_immutable(self):
3016         (newkids, caps) = self._create_immutable_children()
3017         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3018         def _after_mkdir(res):
3019             self.failUnless(res.startswith("URI:DIR"), res)
3020             n = self.s.create_node_from_uri(res)
3021             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3022             d2.addCallback(lambda ign:
3023                            self.failUnlessROChildURIIs(n, u"child-imm",
3024                                                           caps['filecap1']))
3025             d2.addCallback(lambda ign:
3026                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3027                                                           caps['unknown_immcap']))
3028             d2.addCallback(lambda ign:
3029                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3030                                                           caps['immdircap']))
3031             d2.addCallback(lambda ign:
3032                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3033                                                           caps['litdircap']))
3034             d2.addCallback(lambda ign:
3035                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3036                                                           caps['emptydircap']))
3037             return d2
3038         d.addCallback(_after_mkdir)
3039         return d
3040
3041     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3042         (newkids, caps) = self._create_initial_children()
3043         d = self.shouldFail2(error.Error,
3044                              "test_POST_mkdir_no_parentdir_immutable_bad",
3045                              "400 Bad Request",
3046                              "needed to be immutable but was not",
3047                              self.POST2,
3048                              "/uri?t=mkdir-immutable",
3049                              simplejson.dumps(newkids))
3050         return d
3051
3052     def test_welcome_page_mkdir_button(self):
3053         # Fetch the welcome page.
3054         d = self.GET("/")
3055         def _after_get_welcome_page(res):
3056             MKDIR_BUTTON_RE = re.compile(
3057                 '<form action="([^"]*)" method="post".*?'
3058                 '<input type="hidden" name="t" value="([^"]*)" />'
3059                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3060                 '<input type="submit" value="Create a directory" />',
3061                 re.I)
3062             mo = MKDIR_BUTTON_RE.search(res)
3063             formaction = mo.group(1)
3064             formt = mo.group(2)
3065             formaname = mo.group(3)
3066             formavalue = mo.group(4)
3067             return (formaction, formt, formaname, formavalue)
3068         d.addCallback(_after_get_welcome_page)
3069         def _after_parse_form(res):
3070             (formaction, formt, formaname, formavalue) = res
3071             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3072         d.addCallback(_after_parse_form)
3073         d.addBoth(self.shouldRedirect, None, statuscode='303')
3074         return d
3075
3076     def test_POST_mkdir_replace(self): # return value?
3077         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3078         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3079         d.addCallback(self.failUnlessNodeKeysAre, [])
3080         return d
3081
3082     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3083         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3084         d.addBoth(self.shouldFail, error.Error,
3085                   "POST_mkdir_no_replace_queryarg",
3086                   "409 Conflict",
3087                   "There was already a child by that name, and you asked me "
3088                   "to not replace it")
3089         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3090         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3091         return d
3092
3093     def test_POST_mkdir_no_replace_field(self): # return value?
3094         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3095                       replace="false")
3096         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3097                   "409 Conflict",
3098                   "There was already a child by that name, and you asked me "
3099                   "to not replace it")
3100         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3101         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3102         return d
3103
3104     def test_POST_mkdir_whendone_field(self):
3105         d = self.POST(self.public_url + "/foo",
3106                       t="mkdir", name="newdir", when_done="/THERE")
3107         d.addBoth(self.shouldRedirect, "/THERE")
3108         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3109         d.addCallback(self.failUnlessNodeKeysAre, [])
3110         return d
3111
3112     def test_POST_mkdir_whendone_queryarg(self):
3113         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3114                       t="mkdir", name="newdir")
3115         d.addBoth(self.shouldRedirect, "/THERE")
3116         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3117         d.addCallback(self.failUnlessNodeKeysAre, [])
3118         return d
3119
3120     def test_POST_bad_t(self):
3121         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
3122                              "POST to a directory with bad t=BOGUS",
3123                              self.POST, self.public_url + "/foo", t="BOGUS")
3124         return d
3125
3126     def test_POST_set_children(self, command_name="set_children"):
3127         contents9, n9, newuri9 = self.makefile(9)
3128         contents10, n10, newuri10 = self.makefile(10)
3129         contents11, n11, newuri11 = self.makefile(11)
3130
3131         reqbody = """{
3132                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3133                                                 "size": 0,
3134                                                 "metadata": {
3135                                                   "ctime": 1002777696.7564139,
3136                                                   "mtime": 1002777696.7564139
3137                                                  }
3138                                                } ],
3139                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3140                                                 "size": 1,
3141                                                 "metadata": {
3142                                                   "ctime": 1002777696.7564139,
3143                                                   "mtime": 1002777696.7564139
3144                                                  }
3145                                                } ],
3146                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3147                                                 "size": 2,
3148                                                 "metadata": {
3149                                                   "ctime": 1002777696.7564139,
3150                                                   "mtime": 1002777696.7564139
3151                                                  }
3152                                                } ]
3153                     }""" % (newuri9, newuri10, newuri11)
3154
3155         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3156
3157         d = client.getPage(url, method="POST", postdata=reqbody)
3158         def _then(res):
3159             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3160             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3161             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3162
3163         d.addCallback(_then)
3164         d.addErrback(self.dump_error)
3165         return d
3166
3167     def test_POST_set_children_with_hyphen(self):
3168         return self.test_POST_set_children(command_name="set-children")
3169
3170     def test_POST_link_uri(self):
3171         contents, n, newuri = self.makefile(8)
3172         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3173         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3174         d.addCallback(lambda res:
3175                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3176                                                       contents))
3177         return d
3178
3179     def test_POST_link_uri_replace(self):
3180         contents, n, newuri = self.makefile(8)
3181         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3182         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3183         d.addCallback(lambda res:
3184                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3185                                                       contents))
3186         return d
3187
3188     def test_POST_link_uri_unknown_bad(self):
3189         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3190         d.addBoth(self.shouldFail, error.Error,
3191                   "POST_link_uri_unknown_bad",
3192                   "400 Bad Request",
3193                   "unknown cap in a write slot")
3194         return d
3195
3196     def test_POST_link_uri_unknown_ro_good(self):
3197         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3198         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3199         return d
3200
3201     def test_POST_link_uri_unknown_imm_good(self):
3202         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3203         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3204         return d
3205
3206     def test_POST_link_uri_no_replace_queryarg(self):
3207         contents, n, newuri = self.makefile(8)
3208         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3209                       name="bar.txt", uri=newuri)
3210         d.addBoth(self.shouldFail, error.Error,
3211                   "POST_link_uri_no_replace_queryarg",
3212                   "409 Conflict",
3213                   "There was already a child by that name, and you asked me "
3214                   "to not replace it")
3215         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3216         d.addCallback(self.failUnlessIsBarDotTxt)
3217         return d
3218
3219     def test_POST_link_uri_no_replace_field(self):
3220         contents, n, newuri = self.makefile(8)
3221         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3222                       name="bar.txt", uri=newuri)
3223         d.addBoth(self.shouldFail, error.Error,
3224                   "POST_link_uri_no_replace_field",
3225                   "409 Conflict",
3226                   "There was already a child by that name, and you asked me "
3227                   "to not replace it")
3228         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3229         d.addCallback(self.failUnlessIsBarDotTxt)
3230         return d
3231
3232     def test_POST_delete(self, command_name='delete'):
3233         d = self._foo_node.list()
3234         def _check_before(children):
3235             self.failUnless(u"bar.txt" in children)
3236         d.addCallback(_check_before)
3237         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3238         d.addCallback(lambda res: self._foo_node.list())
3239         def _check_after(children):
3240             self.failIf(u"bar.txt" in children)
3241         d.addCallback(_check_after)
3242         return d
3243
3244     def test_POST_unlink(self):
3245         return self.test_POST_delete(command_name='unlink')
3246
3247     def test_POST_rename_file(self):
3248         d = self.POST(self.public_url + "/foo", t="rename",
3249                       from_name="bar.txt", to_name='wibble.txt')
3250         d.addCallback(lambda res:
3251                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3252         d.addCallback(lambda res:
3253                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3254         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3255         d.addCallback(self.failUnlessIsBarDotTxt)
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3257         d.addCallback(self.failUnlessIsBarJSON)
3258         return d
3259
3260     def test_POST_rename_file_redundant(self):
3261         d = self.POST(self.public_url + "/foo", t="rename",
3262                       from_name="bar.txt", to_name='bar.txt')
3263         d.addCallback(lambda res:
3264                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3265         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3266         d.addCallback(self.failUnlessIsBarDotTxt)
3267         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3268         d.addCallback(self.failUnlessIsBarJSON)
3269         return d
3270
3271     def test_POST_rename_file_replace(self):
3272         # rename a file and replace a directory with it
3273         d = self.POST(self.public_url + "/foo", t="rename",
3274                       from_name="bar.txt", to_name='empty')
3275         d.addCallback(lambda res:
3276                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3277         d.addCallback(lambda res:
3278                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3279         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3280         d.addCallback(self.failUnlessIsBarDotTxt)
3281         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3282         d.addCallback(self.failUnlessIsBarJSON)
3283         return d
3284
3285     def test_POST_rename_file_no_replace_queryarg(self):
3286         # rename a file and replace a directory with it
3287         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3288                       from_name="bar.txt", to_name='empty')
3289         d.addBoth(self.shouldFail, error.Error,
3290                   "POST_rename_file_no_replace_queryarg",
3291                   "409 Conflict",
3292                   "There was already a child by that name, and you asked me "
3293                   "to not replace it")
3294         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3295         d.addCallback(self.failUnlessIsEmptyJSON)
3296         return d
3297
3298     def test_POST_rename_file_no_replace_field(self):
3299         # rename a file and replace a directory with it
3300         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3301                       from_name="bar.txt", to_name='empty')
3302         d.addBoth(self.shouldFail, error.Error,
3303                   "POST_rename_file_no_replace_field",
3304                   "409 Conflict",
3305                   "There was already a child by that name, and you asked me "
3306                   "to not replace it")
3307         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3308         d.addCallback(self.failUnlessIsEmptyJSON)
3309         return d
3310
3311     def failUnlessIsEmptyJSON(self, res):
3312         data = simplejson.loads(res)
3313         self.failUnlessEqual(data[0], "dirnode", data)
3314         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3315
3316     def test_POST_rename_file_slash_fail(self):
3317         d = self.POST(self.public_url + "/foo", t="rename",
3318                       from_name="bar.txt", to_name='kirk/spock.txt')
3319         d.addBoth(self.shouldFail, error.Error,
3320                   "test_POST_rename_file_slash_fail",
3321                   "400 Bad Request",
3322                   "to_name= may not contain a slash",
3323                   )
3324         d.addCallback(lambda res:
3325                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3326         return d
3327
3328     def test_POST_rename_dir(self):
3329         d = self.POST(self.public_url, t="rename",
3330                       from_name="foo", to_name='plunk')
3331         d.addCallback(lambda res:
3332                       self.failIfNodeHasChild(self.public_root, u"foo"))
3333         d.addCallback(lambda res:
3334                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3335         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3336         d.addCallback(self.failUnlessIsFooJSON)
3337         return d
3338
3339     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3340         """ If target is not None then the redirection has to go to target.  If
3341         statuscode is not None then the redirection has to be accomplished with
3342         that HTTP status code."""
3343         if not isinstance(res, failure.Failure):
3344             to_where = (target is None) and "somewhere" or ("to " + target)
3345             self.fail("%s: we were expecting to get redirected %s, not get an"
3346                       " actual page: %s" % (which, to_where, res))
3347         res.trap(error.PageRedirect)
3348         if statuscode is not None:
3349             self.failUnlessReallyEqual(res.value.status, statuscode,
3350                                        "%s: not a redirect" % which)
3351         if target is not None:
3352             # the PageRedirect does not seem to capture the uri= query arg
3353             # properly, so we can't check for it.
3354             realtarget = self.webish_url + target
3355             self.failUnlessReallyEqual(res.value.location, realtarget,
3356                                        "%s: wrong target" % which)
3357         return res.value.location
3358
3359     def test_GET_URI_form(self):
3360         base = "/uri?uri=%s" % self._bar_txt_uri
3361         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3362         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3363         d = self.GET(base)
3364         d.addBoth(self.shouldRedirect, targetbase)
3365         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3366         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3367         d.addCallback(lambda res: self.GET(base+"&t=json"))
3368         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3369         d.addCallback(self.log, "about to get file by uri")
3370         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3371         d.addCallback(self.failUnlessIsBarDotTxt)
3372         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3373         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3374                                            followRedirect=True))
3375         d.addCallback(self.failUnlessIsFooJSON)
3376         d.addCallback(self.log, "got dir by uri")
3377
3378         return d
3379
3380     def test_GET_URI_form_bad(self):
3381         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3382                              "400 Bad Request", "GET /uri requires uri=",
3383                              self.GET, "/uri")
3384         return d
3385
3386     def test_GET_rename_form(self):
3387         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3388                      followRedirect=True)
3389         def _check(res):
3390             self.failUnless('name="when_done" value="."' in res, res)
3391             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3392         d.addCallback(_check)
3393         return d
3394
3395     def log(self, res, msg):
3396         #print "MSG: %s  RES: %s" % (msg, res)
3397         log.msg(msg)
3398         return res
3399
3400     def test_GET_URI_URL(self):
3401         base = "/uri/%s" % self._bar_txt_uri
3402         d = self.GET(base)
3403         d.addCallback(self.failUnlessIsBarDotTxt)
3404         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3405         d.addCallback(self.failUnlessIsBarDotTxt)
3406         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3407         d.addCallback(self.failUnlessIsBarDotTxt)
3408         return d
3409
3410     def test_GET_URI_URL_dir(self):
3411         base = "/uri/%s?t=json" % self._foo_uri
3412         d = self.GET(base)
3413         d.addCallback(self.failUnlessIsFooJSON)
3414         return d
3415
3416     def test_GET_URI_URL_missing(self):
3417         base = "/uri/%s" % self._bad_file_uri
3418         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3419                                  http.GONE, None, "NotEnoughSharesError",
3420                                  self.GET, base)
3421         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3422         # here? we must arrange for a download to fail after target.open()
3423         # has been called, and then inspect the response to see that it is
3424         # shorter than we expected.
3425         return d
3426
3427     def test_PUT_DIRURL_uri(self):
3428         d = self.s.create_dirnode()
3429         def _made_dir(dn):
3430             new_uri = dn.get_uri()
3431             # replace /foo with a new (empty) directory
3432             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3433             d.addCallback(lambda res:
3434                           self.failUnlessReallyEqual(res.strip(), new_uri))
3435             d.addCallback(lambda res:
3436                           self.failUnlessRWChildURIIs(self.public_root,
3437                                                       u"foo",
3438                                                       new_uri))
3439             return d
3440         d.addCallback(_made_dir)
3441         return d
3442
3443     def test_PUT_DIRURL_uri_noreplace(self):
3444         d = self.s.create_dirnode()
3445         def _made_dir(dn):
3446             new_uri = dn.get_uri()
3447             # replace /foo with a new (empty) directory, but ask that
3448             # replace=false, so it should fail
3449             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3450                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3451                                  self.PUT,
3452                                  self.public_url + "/foo?t=uri&replace=false",
3453                                  new_uri)
3454             d.addCallback(lambda res:
3455                           self.failUnlessRWChildURIIs(self.public_root,
3456                                                       u"foo",
3457                                                       self._foo_uri))
3458             return d
3459         d.addCallback(_made_dir)
3460         return d
3461
3462     def test_PUT_DIRURL_bad_t(self):
3463         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3464                                  "400 Bad Request", "PUT to a directory",
3465                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
3466         d.addCallback(lambda res:
3467                       self.failUnlessRWChildURIIs(self.public_root,
3468                                                   u"foo",
3469                                                   self._foo_uri))
3470         return d
3471
3472     def test_PUT_NEWFILEURL_uri(self):
3473         contents, n, new_uri = self.makefile(8)
3474         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3475         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3476         d.addCallback(lambda res:
3477                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3478                                                       contents))
3479         return d
3480
3481     def test_PUT_NEWFILEURL_mdmf(self):
3482         new_contents = self.NEWFILE_CONTENTS * 300000
3483         d = self.PUT(self.public_url + \
3484                      "/foo/mdmf.txt?format=mdmf",
3485                      new_contents)
3486         d.addCallback(lambda ignored:
3487             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3488         def _got_json(json):
3489             data = simplejson.loads(json)
3490             data = data[1]
3491             self.failUnlessIn("format", data)
3492             self.failUnlessEqual(data["format"], "mdmf")
3493             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3494             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3495         d.addCallback(_got_json)
3496         return d
3497
3498     def test_PUT_NEWFILEURL_sdmf(self):
3499         new_contents = self.NEWFILE_CONTENTS * 300000
3500         d = self.PUT(self.public_url + \
3501                      "/foo/sdmf.txt?format=sdmf",
3502                      new_contents)
3503         d.addCallback(lambda ignored:
3504             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3505         def _got_json(json):
3506             data = simplejson.loads(json)
3507             data = data[1]
3508             self.failUnlessIn("format", data)
3509             self.failUnlessEqual(data["format"], "sdmf")
3510         d.addCallback(_got_json)
3511         return d
3512
3513     def test_PUT_NEWFILEURL_bad_format(self):
3514        new_contents = self.NEWFILE_CONTENTS * 300000
3515        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3516                                    400, "Bad Request", "Unknown format: foo",
3517                                    self.PUT, self.public_url + \
3518                                    "/foo/foo.txt?format=foo",
3519                                    new_contents)
3520
3521     def test_PUT_NEWFILEURL_uri_replace(self):
3522         contents, n, new_uri = self.makefile(8)
3523         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3524         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3525         d.addCallback(lambda res:
3526                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3527                                                       contents))
3528         return d
3529
3530     def test_PUT_NEWFILEURL_uri_no_replace(self):
3531         contents, n, new_uri = self.makefile(8)
3532         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3533         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
3534                   "409 Conflict",
3535                   "There was already a child by that name, and you asked me "
3536                   "to not replace it")
3537         return d
3538
3539     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3540         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3541         d.addBoth(self.shouldFail, error.Error,
3542                   "POST_put_uri_unknown_bad",
3543                   "400 Bad Request",
3544                   "unknown cap in a write slot")
3545         return d
3546
3547     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3548         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3549         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3550                       u"put-future-ro.txt")
3551         return d
3552
3553     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3554         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3555         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3556                       u"put-future-imm.txt")
3557         return d
3558
3559     def test_PUT_NEWFILE_URI(self):
3560         file_contents = "New file contents here\n"
3561         d = self.PUT("/uri", file_contents)
3562         def _check(uri):
3563             assert isinstance(uri, str), uri
3564             self.failUnless(uri in FakeCHKFileNode.all_contents)
3565             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3566                                        file_contents)
3567             return self.GET("/uri/%s" % uri)
3568         d.addCallback(_check)
3569         def _check2(res):
3570             self.failUnlessReallyEqual(res, file_contents)
3571         d.addCallback(_check2)
3572         return d
3573
3574     def test_PUT_NEWFILE_URI_not_mutable(self):
3575         file_contents = "New file contents here\n"
3576         d = self.PUT("/uri?mutable=false", file_contents)
3577         def _check(uri):
3578             assert isinstance(uri, str), uri
3579             self.failUnless(uri in FakeCHKFileNode.all_contents)
3580             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3581                                        file_contents)
3582             return self.GET("/uri/%s" % uri)
3583         d.addCallback(_check)
3584         def _check2(res):
3585             self.failUnlessReallyEqual(res, file_contents)
3586         d.addCallback(_check2)
3587         return d
3588
3589     def test_PUT_NEWFILE_URI_only_PUT(self):
3590         d = self.PUT("/uri?t=bogus", "")
3591         d.addBoth(self.shouldFail, error.Error,
3592                   "PUT_NEWFILE_URI_only_PUT",
3593                   "400 Bad Request",
3594                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3595         return d
3596
3597     def test_PUT_NEWFILE_URI_mutable(self):
3598         file_contents = "New file contents here\n"
3599         d = self.PUT("/uri?mutable=true", file_contents)
3600         def _check1(filecap):
3601             filecap = filecap.strip()
3602             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3603             self.filecap = filecap
3604             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3605             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3606             n = self.s.create_node_from_uri(filecap)
3607             return n.download_best_version()
3608         d.addCallback(_check1)
3609         def _check2(data):
3610             self.failUnlessReallyEqual(data, file_contents)
3611             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3612         d.addCallback(_check2)
3613         def _check3(res):
3614             self.failUnlessReallyEqual(res, file_contents)
3615         d.addCallback(_check3)
3616         return d
3617
3618     def test_PUT_mkdir(self):
3619         d = self.PUT("/uri?t=mkdir", "")
3620         def _check(uri):
3621             n = self.s.create_node_from_uri(uri.strip())
3622             d2 = self.failUnlessNodeKeysAre(n, [])
3623             d2.addCallback(lambda res:
3624                            self.GET("/uri/%s?t=json" % uri))
3625             return d2
3626         d.addCallback(_check)
3627         d.addCallback(self.failUnlessIsEmptyJSON)
3628         return d
3629
3630     def test_PUT_mkdir_mdmf(self):
3631         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3632         def _got(res):
3633             u = uri.from_string(res)
3634             # Check that this is an MDMF writecap
3635             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3636         d.addCallback(_got)
3637         return d
3638
3639     def test_PUT_mkdir_sdmf(self):
3640         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3641         def _got(res):
3642             u = uri.from_string(res)
3643             self.failUnlessIsInstance(u, uri.DirectoryURI)
3644         d.addCallback(_got)
3645         return d
3646
3647     def test_PUT_mkdir_bad_format(self):
3648         return self.shouldHTTPError("PUT_mkdir_bad_format",
3649                                     400, "Bad Request", "Unknown format: foo",
3650                                     self.PUT, "/uri?t=mkdir&format=foo",
3651                                     "")
3652
3653     def test_POST_check(self):
3654         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3655         def _done(res):
3656             # this returns a string form of the results, which are probably
3657             # None since we're using fake filenodes.
3658             # TODO: verify that the check actually happened, by changing
3659             # FakeCHKFileNode to count how many times .check() has been
3660             # called.
3661             pass
3662         d.addCallback(_done)
3663         return d
3664
3665
3666     def test_PUT_update_at_offset(self):
3667         file_contents = "test file" * 100000 # about 900 KiB
3668         d = self.PUT("/uri?mutable=true", file_contents)
3669         def _then(filecap):
3670             self.filecap = filecap
3671             new_data = file_contents[:100]
3672             new = "replaced and so on"
3673             new_data += new
3674             new_data += file_contents[len(new_data):]
3675             assert len(new_data) == len(file_contents)
3676             self.new_data = new_data
3677         d.addCallback(_then)
3678         d.addCallback(lambda ignored:
3679             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3680                      "replaced and so on"))
3681         def _get_data(filecap):
3682             n = self.s.create_node_from_uri(filecap)
3683             return n.download_best_version()
3684         d.addCallback(_get_data)
3685         d.addCallback(lambda results:
3686             self.failUnlessEqual(results, self.new_data))
3687         # Now try appending things to the file
3688         d.addCallback(lambda ignored:
3689             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3690                      "puppies" * 100))
3691         d.addCallback(_get_data)
3692         d.addCallback(lambda results:
3693             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3694         # and try replacing the beginning of the file
3695         d.addCallback(lambda ignored:
3696             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3697         d.addCallback(_get_data)
3698         d.addCallback(lambda results:
3699             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3700         return d
3701
3702     def test_PUT_update_at_invalid_offset(self):
3703         file_contents = "test file" * 100000 # about 900 KiB
3704         d = self.PUT("/uri?mutable=true", file_contents)
3705         def _then(filecap):
3706             self.filecap = filecap
3707         d.addCallback(_then)
3708         # Negative offsets should cause an error.
3709         d.addCallback(lambda ignored:
3710             self.shouldHTTPError("test mutable invalid offset negative",
3711                                  400, "Bad Request",
3712                                  "Invalid offset",
3713                                  self.PUT,
3714                                  "/uri/%s?offset=-1" % self.filecap,
3715                                  "foo"))
3716         return d
3717
3718     def test_PUT_update_at_offset_immutable(self):
3719         file_contents = "Test file" * 100000
3720         d = self.PUT("/uri", file_contents)
3721         def _then(filecap):
3722             self.filecap = filecap
3723         d.addCallback(_then)
3724         d.addCallback(lambda ignored:
3725             self.shouldHTTPError("test immutable update",
3726                                  400, "Bad Request",
3727                                  "immutable",
3728                                  self.PUT,
3729                                  "/uri/%s?offset=50" % self.filecap,
3730                                  "foo"))
3731         return d
3732
3733
3734     def test_bad_method(self):
3735         url = self.webish_url + self.public_url + "/foo/bar.txt"
3736         d = self.shouldHTTPError("test_bad_method",
3737                                  501, "Not Implemented",
3738                                  "I don't know how to treat a BOGUS request.",
3739                                  client.getPage, url, method="BOGUS")
3740         return d
3741
3742     def test_short_url(self):
3743         url = self.webish_url + "/uri"
3744         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3745                                  "I don't know how to treat a DELETE request.",
3746                                  client.getPage, url, method="DELETE")
3747         return d
3748
3749     def test_ophandle_bad(self):
3750         url = self.webish_url + "/operations/bogus?t=status"
3751         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3752                                  "unknown/expired handle 'bogus'",
3753                                  client.getPage, url)
3754         return d
3755
3756     def test_ophandle_cancel(self):
3757         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3758                       followRedirect=True)
3759         d.addCallback(lambda ignored:
3760                       self.GET("/operations/128?t=status&output=JSON"))
3761         def _check1(res):
3762             data = simplejson.loads(res)
3763             self.failUnless("finished" in data, res)
3764             monitor = self.ws.root.child_operations.handles["128"][0]
3765             d = self.POST("/operations/128?t=cancel&output=JSON")
3766             def _check2(res):
3767                 data = simplejson.loads(res)
3768                 self.failUnless("finished" in data, res)
3769                 # t=cancel causes the handle to be forgotten
3770                 self.failUnless(monitor.is_cancelled())
3771             d.addCallback(_check2)
3772             return d
3773         d.addCallback(_check1)
3774         d.addCallback(lambda ignored:
3775                       self.shouldHTTPError("test_ophandle_cancel",
3776                                            404, "404 Not Found",
3777                                            "unknown/expired handle '128'",
3778                                            self.GET,
3779                                            "/operations/128?t=status&output=JSON"))
3780         return d
3781
3782     def test_ophandle_retainfor(self):
3783         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3784                       followRedirect=True)
3785         d.addCallback(lambda ignored:
3786                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3787         def _check1(res):
3788             data = simplejson.loads(res)
3789             self.failUnless("finished" in data, res)
3790         d.addCallback(_check1)
3791         # the retain-for=0 will cause the handle to be expired very soon
3792         d.addCallback(lambda ign:
3793             self.clock.advance(2.0))
3794         d.addCallback(lambda ignored:
3795                       self.shouldHTTPError("test_ophandle_retainfor",
3796                                            404, "404 Not Found",
3797                                            "unknown/expired handle '129'",
3798                                            self.GET,
3799                                            "/operations/129?t=status&output=JSON"))
3800         return d
3801
3802     def test_ophandle_release_after_complete(self):
3803         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3804                       followRedirect=True)
3805         d.addCallback(self.wait_for_operation, "130")
3806         d.addCallback(lambda ignored:
3807                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3808         # the release-after-complete=true will cause the handle to be expired
3809         d.addCallback(lambda ignored:
3810                       self.shouldHTTPError("test_ophandle_release_after_complete",
3811                                            404, "404 Not Found",
3812                                            "unknown/expired handle '130'",
3813                                            self.GET,
3814                                            "/operations/130?t=status&output=JSON"))
3815         return d
3816
3817     def test_uncollected_ophandle_expiration(self):
3818         # uncollected ophandles should expire after 4 days
3819         def _make_uncollected_ophandle(ophandle):
3820             d = self.POST(self.public_url +
3821                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3822                           followRedirect=False)
3823             # When we start the operation, the webapi server will want
3824             # to redirect us to the page for the ophandle, so we get
3825             # confirmation that the operation has started. If the
3826             # manifest operation has finished by the time we get there,
3827             # following that redirect (by setting followRedirect=True
3828             # above) has the side effect of collecting the ophandle that
3829             # we've just created, which means that we can't use the
3830             # ophandle to test the uncollected timeout anymore. So,
3831             # instead, catch the 302 here and don't follow it.
3832             d.addBoth(self.should302, "uncollected_ophandle_creation")
3833             return d
3834         # Create an ophandle, don't collect it, then advance the clock by
3835         # 4 days - 1 second and make sure that the ophandle is still there.
3836         d = _make_uncollected_ophandle(131)
3837         d.addCallback(lambda ign:
3838             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3839         d.addCallback(lambda ign:
3840             self.GET("/operations/131?t=status&output=JSON"))
3841         def _check1(res):
3842             data = simplejson.loads(res)
3843             self.failUnless("finished" in data, res)
3844         d.addCallback(_check1)
3845         # Create an ophandle, don't collect it, then try to collect it
3846         # after 4 days. It should be gone.
3847         d.addCallback(lambda ign:
3848             _make_uncollected_ophandle(132))
3849         d.addCallback(lambda ign:
3850             self.clock.advance(96*60*60))
3851         d.addCallback(lambda ign:
3852             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3853                                  404, "404 Not Found",
3854                                  "unknown/expired handle '132'",
3855                                  self.GET,
3856                                  "/operations/132?t=status&output=JSON"))
3857         return d
3858
3859     def test_collected_ophandle_expiration(self):
3860         # collected ophandles should expire after 1 day
3861         def _make_collected_ophandle(ophandle):
3862             d = self.POST(self.public_url +
3863                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3864                           followRedirect=True)
3865             # By following the initial redirect, we collect the ophandle
3866             # we've just created.
3867             return d
3868         # Create a collected ophandle, then collect it after 23 hours
3869         # and 59 seconds to make sure that it is still there.
3870         d = _make_collected_ophandle(133)
3871         d.addCallback(lambda ign:
3872             self.clock.advance((24*60*60) - 1))
3873         d.addCallback(lambda ign:
3874             self.GET("/operations/133?t=status&output=JSON"))
3875         def _check1(res):
3876             data = simplejson.loads(res)
3877             self.failUnless("finished" in data, res)
3878         d.addCallback(_check1)
3879         # Create another uncollected ophandle, then try to collect it
3880         # after 24 hours to make sure that it is gone.
3881         d.addCallback(lambda ign:
3882             _make_collected_ophandle(134))
3883         d.addCallback(lambda ign:
3884             self.clock.advance(24*60*60))
3885         d.addCallback(lambda ign:
3886             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3887                                  404, "404 Not Found",
3888                                  "unknown/expired handle '134'",
3889                                  self.GET,
3890                                  "/operations/134?t=status&output=JSON"))
3891         return d
3892
3893     def test_incident(self):
3894         d = self.POST("/report_incident", details="eek")
3895         def _done(res):
3896             self.failUnless("Thank you for your report!" in res, res)
3897         d.addCallback(_done)
3898         return d
3899
3900     def test_static(self):
3901         webdir = os.path.join(self.staticdir, "subdir")
3902         fileutil.make_dirs(webdir)
3903         f = open(os.path.join(webdir, "hello.txt"), "wb")
3904         f.write("hello")
3905         f.close()
3906
3907         d = self.GET("/static/subdir/hello.txt")
3908         def _check(res):
3909             self.failUnlessReallyEqual(res, "hello")
3910         d.addCallback(_check)
3911         return d
3912
3913
3914 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3915     def test_load_file(self):
3916         # This will raise an exception unless a well-formed XML file is found under that name.
3917         common.getxmlfile('directory.xhtml').load()
3918
3919     def test_parse_replace_arg(self):
3920         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3921         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3922         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3923                                    "only-files")
3924         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3925                         common.parse_replace_arg, "only_fles")
3926
3927     def test_abbreviate_time(self):
3928         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3929         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3930         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3931         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3932         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3933         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3934
3935     def test_compute_rate(self):
3936         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3937         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3938         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3939         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3940         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3941         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3942         self.shouldFail(AssertionError, "test_compute_rate", "",
3943                         common.compute_rate, -100, 10)
3944         self.shouldFail(AssertionError, "test_compute_rate", "",
3945                         common.compute_rate, 100, -10)
3946
3947         # Sanity check
3948         rate = common.compute_rate(10*1000*1000, 1)
3949         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3950
3951     def test_abbreviate_rate(self):
3952         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3953         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3954         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3955         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3956
3957     def test_abbreviate_size(self):
3958         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3959         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3960         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3961         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3962         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3963
3964     def test_plural(self):
3965         def convert(s):
3966             return "%d second%s" % (s, status.plural(s))
3967         self.failUnlessReallyEqual(convert(0), "0 seconds")
3968         self.failUnlessReallyEqual(convert(1), "1 second")
3969         self.failUnlessReallyEqual(convert(2), "2 seconds")
3970         def convert2(s):
3971             return "has share%s: %s" % (status.plural(s), ",".join(s))
3972         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3973         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3974         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3975
3976
3977 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3978
3979     def CHECK(self, ign, which, args, clientnum=0):
3980         fileurl = self.fileurls[which]
3981         url = fileurl + "?" + args
3982         return self.GET(url, method="POST", clientnum=clientnum)
3983
3984     def test_filecheck(self):
3985         self.basedir = "web/Grid/filecheck"
3986         self.set_up_grid()
3987         c0 = self.g.clients[0]
3988         self.uris = {}
3989         DATA = "data" * 100
3990         d = c0.upload(upload.Data(DATA, convergence=""))
3991         def _stash_uri(ur, which):
3992             self.uris[which] = ur.uri
3993         d.addCallback(_stash_uri, "good")
3994         d.addCallback(lambda ign:
3995                       c0.upload(upload.Data(DATA+"1", convergence="")))
3996         d.addCallback(_stash_uri, "sick")
3997         d.addCallback(lambda ign:
3998                       c0.upload(upload.Data(DATA+"2", convergence="")))
3999         d.addCallback(_stash_uri, "dead")
4000         def _stash_mutable_uri(n, which):
4001             self.uris[which] = n.get_uri()
4002             assert isinstance(self.uris[which], str)
4003         d.addCallback(lambda ign:
4004             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4005         d.addCallback(_stash_mutable_uri, "corrupt")
4006         d.addCallback(lambda ign:
4007                       c0.upload(upload.Data("literal", convergence="")))
4008         d.addCallback(_stash_uri, "small")
4009         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4010         d.addCallback(_stash_mutable_uri, "smalldir")
4011
4012         def _compute_fileurls(ignored):
4013             self.fileurls = {}
4014             for which in self.uris:
4015                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4016         d.addCallback(_compute_fileurls)
4017
4018         def _clobber_shares(ignored):
4019             good_shares = self.find_uri_shares(self.uris["good"])
4020             self.failUnlessReallyEqual(len(good_shares), 10)
4021             sick_shares = self.find_uri_shares(self.uris["sick"])
4022             os.unlink(sick_shares[0][2])
4023             dead_shares = self.find_uri_shares(self.uris["dead"])
4024             for i in range(1, 10):
4025                 os.unlink(dead_shares[i][2])
4026             c_shares = self.find_uri_shares(self.uris["corrupt"])
4027             cso = CorruptShareOptions()
4028             cso.stdout = StringIO()
4029             cso.parseOptions([c_shares[0][2]])
4030             corrupt_share(cso)
4031         d.addCallback(_clobber_shares)
4032
4033         d.addCallback(self.CHECK, "good", "t=check")
4034         def _got_html_good(res):
4035             self.failUnless("Healthy" in res, res)
4036             self.failIf("Not Healthy" in res, res)
4037         d.addCallback(_got_html_good)
4038         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4039         def _got_html_good_return_to(res):
4040             self.failUnless("Healthy" in res, res)
4041             self.failIf("Not Healthy" in res, res)
4042             self.failUnless('<a href="somewhere">Return to file'
4043                             in res, res)
4044         d.addCallback(_got_html_good_return_to)
4045         d.addCallback(self.CHECK, "good", "t=check&output=json")
4046         def _got_json_good(res):
4047             r = simplejson.loads(res)
4048             self.failUnlessEqual(r["summary"], "Healthy")
4049             self.failUnless(r["results"]["healthy"])
4050             self.failIf(r["results"]["needs-rebalancing"])
4051             self.failUnless(r["results"]["recoverable"])
4052         d.addCallback(_got_json_good)
4053
4054         d.addCallback(self.CHECK, "small", "t=check")
4055         def _got_html_small(res):
4056             self.failUnless("Literal files are always healthy" in res, res)
4057             self.failIf("Not Healthy" in res, res)
4058         d.addCallback(_got_html_small)
4059         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4060         def _got_html_small_return_to(res):
4061             self.failUnless("Literal files are always healthy" in res, res)
4062             self.failIf("Not Healthy" in res, res)
4063             self.failUnless('<a href="somewhere">Return to file'
4064                             in res, res)
4065         d.addCallback(_got_html_small_return_to)
4066         d.addCallback(self.CHECK, "small", "t=check&output=json")
4067         def _got_json_small(res):
4068             r = simplejson.loads(res)
4069             self.failUnlessEqual(r["storage-index"], "")
4070             self.failUnless(r["results"]["healthy"])
4071         d.addCallback(_got_json_small)
4072
4073         d.addCallback(self.CHECK, "smalldir", "t=check")
4074         def _got_html_smalldir(res):
4075             self.failUnless("Literal files are always healthy" in res, res)
4076             self.failIf("Not Healthy" in res, res)
4077         d.addCallback(_got_html_smalldir)
4078         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4079         def _got_json_smalldir(res):
4080             r = simplejson.loads(res)
4081             self.failUnlessEqual(r["storage-index"], "")
4082             self.failUnless(r["results"]["healthy"])
4083         d.addCallback(_got_json_smalldir)
4084
4085         d.addCallback(self.CHECK, "sick", "t=check")
4086         def _got_html_sick(res):
4087             self.failUnless("Not Healthy" in res, res)
4088         d.addCallback(_got_html_sick)
4089         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4090         def _got_json_sick(res):
4091             r = simplejson.loads(res)
4092             self.failUnlessEqual(r["summary"],
4093                                  "Not Healthy: 9 shares (enc 3-of-10)")
4094             self.failIf(r["results"]["healthy"])
4095             self.failIf(r["results"]["needs-rebalancing"])
4096             self.failUnless(r["results"]["recoverable"])
4097         d.addCallback(_got_json_sick)
4098
4099         d.addCallback(self.CHECK, "dead", "t=check")
4100         def _got_html_dead(res):
4101             self.failUnless("Not Healthy" in res, res)
4102         d.addCallback(_got_html_dead)
4103         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4104         def _got_json_dead(res):
4105             r = simplejson.loads(res)
4106             self.failUnlessEqual(r["summary"],
4107                                  "Not Healthy: 1 shares (enc 3-of-10)")
4108             self.failIf(r["results"]["healthy"])
4109             self.failIf(r["results"]["needs-rebalancing"])
4110             self.failIf(r["results"]["recoverable"])
4111         d.addCallback(_got_json_dead)
4112
4113         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4114         def _got_html_corrupt(res):
4115             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4116         d.addCallback(_got_html_corrupt)
4117         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4118         def _got_json_corrupt(res):
4119             r = simplejson.loads(res)
4120             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4121                             r["summary"])
4122             self.failIf(r["results"]["healthy"])
4123             self.failUnless(r["results"]["recoverable"])
4124             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4125             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4126         d.addCallback(_got_json_corrupt)
4127
4128         d.addErrback(self.explain_web_error)
4129         return d
4130
4131     def test_repair_html(self):
4132         self.basedir = "web/Grid/repair_html"
4133         self.set_up_grid()
4134         c0 = self.g.clients[0]
4135         self.uris = {}
4136         DATA = "data" * 100
4137         d = c0.upload(upload.Data(DATA, convergence=""))
4138         def _stash_uri(ur, which):
4139             self.uris[which] = ur.uri
4140         d.addCallback(_stash_uri, "good")
4141         d.addCallback(lambda ign:
4142                       c0.upload(upload.Data(DATA+"1", convergence="")))
4143         d.addCallback(_stash_uri, "sick")
4144         d.addCallback(lambda ign:
4145                       c0.upload(upload.Data(DATA+"2", convergence="")))
4146         d.addCallback(_stash_uri, "dead")
4147         def _stash_mutable_uri(n, which):
4148             self.uris[which] = n.get_uri()
4149             assert isinstance(self.uris[which], str)
4150         d.addCallback(lambda ign:
4151             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4152         d.addCallback(_stash_mutable_uri, "corrupt")
4153
4154         def _compute_fileurls(ignored):
4155             self.fileurls = {}
4156             for which in self.uris:
4157                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4158         d.addCallback(_compute_fileurls)
4159
4160         def _clobber_shares(ignored):
4161             good_shares = self.find_uri_shares(self.uris["good"])
4162             self.failUnlessReallyEqual(len(good_shares), 10)
4163             sick_shares = self.find_uri_shares(self.uris["sick"])
4164             os.unlink(sick_shares[0][2])
4165             dead_shares = self.find_uri_shares(self.uris["dead"])
4166             for i in range(1, 10):
4167                 os.unlink(dead_shares[i][2])
4168             c_shares = self.find_uri_shares(self.uris["corrupt"])
4169             cso = CorruptShareOptions()
4170             cso.stdout = StringIO()
4171             cso.parseOptions([c_shares[0][2]])
4172             corrupt_share(cso)
4173         d.addCallback(_clobber_shares)
4174
4175         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4176         def _got_html_good(res):
4177             self.failUnless("Healthy" in res, res)
4178             self.failIf("Not Healthy" in res, res)
4179             self.failUnless("No repair necessary" in res, res)
4180         d.addCallback(_got_html_good)
4181
4182         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4183         def _got_html_sick(res):
4184             self.failUnless("Healthy : healthy" in res, res)
4185             self.failIf("Not Healthy" in res, res)
4186             self.failUnless("Repair successful" in res, res)
4187         d.addCallback(_got_html_sick)
4188
4189         # repair of a dead file will fail, of course, but it isn't yet
4190         # clear how this should be reported. Right now it shows up as
4191         # a "410 Gone".
4192         #
4193         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4194         #def _got_html_dead(res):
4195         #    print res
4196         #    self.failUnless("Healthy : healthy" in res, res)
4197         #    self.failIf("Not Healthy" in res, res)
4198         #    self.failUnless("No repair necessary" in res, res)
4199         #d.addCallback(_got_html_dead)
4200
4201         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4202         def _got_html_corrupt(res):
4203             self.failUnless("Healthy : Healthy" in res, res)
4204             self.failIf("Not Healthy" in res, res)
4205             self.failUnless("Repair successful" in res, res)
4206         d.addCallback(_got_html_corrupt)
4207
4208         d.addErrback(self.explain_web_error)
4209         return d
4210
4211     def test_repair_json(self):
4212         self.basedir = "web/Grid/repair_json"
4213         self.set_up_grid()
4214         c0 = self.g.clients[0]
4215         self.uris = {}
4216         DATA = "data" * 100
4217         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4218         def _stash_uri(ur, which):
4219             self.uris[which] = ur.uri
4220         d.addCallback(_stash_uri, "sick")
4221
4222         def _compute_fileurls(ignored):
4223             self.fileurls = {}
4224             for which in self.uris:
4225                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4226         d.addCallback(_compute_fileurls)
4227
4228         def _clobber_shares(ignored):
4229             sick_shares = self.find_uri_shares(self.uris["sick"])
4230             os.unlink(sick_shares[0][2])
4231         d.addCallback(_clobber_shares)
4232
4233         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4234         def _got_json_sick(res):
4235             r = simplejson.loads(res)
4236             self.failUnlessReallyEqual(r["repair-attempted"], True)
4237             self.failUnlessReallyEqual(r["repair-successful"], True)
4238             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4239                                  "Not Healthy: 9 shares (enc 3-of-10)")
4240             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4241             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4242             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4243         d.addCallback(_got_json_sick)
4244
4245         d.addErrback(self.explain_web_error)
4246         return d
4247
4248     def test_unknown(self, immutable=False):
4249         self.basedir = "web/Grid/unknown"
4250         if immutable:
4251             self.basedir = "web/Grid/unknown-immutable"
4252
4253         self.set_up_grid()
4254         c0 = self.g.clients[0]
4255         self.uris = {}
4256         self.fileurls = {}
4257
4258         # the future cap format may contain slashes, which must be tolerated
4259         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4260                                                            safe="")
4261
4262         if immutable:
4263             name = u"future-imm"
4264             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4265             d = c0.create_immutable_dirnode({name: (future_node, {})})
4266         else:
4267             name = u"future"
4268             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4269             d = c0.create_dirnode()
4270
4271         def _stash_root_and_create_file(n):
4272             self.rootnode = n
4273             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4274             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4275             if not immutable:
4276                 return self.rootnode.set_node(name, future_node)
4277         d.addCallback(_stash_root_and_create_file)
4278
4279         # make sure directory listing tolerates unknown nodes
4280         d.addCallback(lambda ign: self.GET(self.rooturl))
4281         def _check_directory_html(res, expected_type_suffix):
4282             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4283                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4284                                  re.DOTALL)
4285             self.failUnless(re.search(pattern, res), res)
4286             # find the More Info link for name, should be relative
4287             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4288             info_url = mo.group(1)
4289             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4290         if immutable:
4291             d.addCallback(_check_directory_html, "-IMM")
4292         else:
4293             d.addCallback(_check_directory_html, "")
4294
4295         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4296         def _check_directory_json(res, expect_rw_uri):
4297             data = simplejson.loads(res)
4298             self.failUnlessEqual(data[0], "dirnode")
4299             f = data[1]["children"][name]
4300             self.failUnlessEqual(f[0], "unknown")
4301             if expect_rw_uri:
4302                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4303             else:
4304                 self.failIfIn("rw_uri", f[1])
4305             if immutable:
4306                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4307             else:
4308                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4309             self.failUnless("metadata" in f[1])
4310         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4311
4312         def _check_info(res, expect_rw_uri, expect_ro_uri):
4313             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4314             if expect_rw_uri:
4315                 self.failUnlessIn(unknown_rwcap, res)
4316             if expect_ro_uri:
4317                 if immutable:
4318                     self.failUnlessIn(unknown_immcap, res)
4319                 else:
4320                     self.failUnlessIn(unknown_rocap, res)
4321             else:
4322                 self.failIfIn(unknown_rocap, res)
4323             self.failIfIn("Raw data as", res)
4324             self.failIfIn("Directory writecap", res)
4325             self.failIfIn("Checker Operations", res)
4326             self.failIfIn("Mutable File Operations", res)
4327             self.failIfIn("Directory Operations", res)
4328
4329         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4330         # why they fail. Possibly related to ticket #922.
4331
4332         d.addCallback(lambda ign: self.GET(expected_info_url))
4333         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4334         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4335         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4336
4337         def _check_json(res, expect_rw_uri):
4338             data = simplejson.loads(res)
4339             self.failUnlessEqual(data[0], "unknown")
4340             if expect_rw_uri:
4341                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4342             else:
4343                 self.failIfIn("rw_uri", data[1])
4344
4345             if immutable:
4346                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4347                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4348             elif expect_rw_uri:
4349                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4350                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4351             else:
4352                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4353                 self.failIf("mutable" in data[1], data[1])
4354
4355             # TODO: check metadata contents
4356             self.failUnless("metadata" in data[1])
4357
4358         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4359         d.addCallback(_check_json, expect_rw_uri=not immutable)
4360
4361         # and make sure that a read-only version of the directory can be
4362         # rendered too. This version will not have unknown_rwcap, whether
4363         # or not future_node was immutable.
4364         d.addCallback(lambda ign: self.GET(self.rourl))
4365         if immutable:
4366             d.addCallback(_check_directory_html, "-IMM")
4367         else:
4368             d.addCallback(_check_directory_html, "-RO")
4369
4370         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4371         d.addCallback(_check_directory_json, expect_rw_uri=False)
4372
4373         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4374         d.addCallback(_check_json, expect_rw_uri=False)
4375
4376         # TODO: check that getting t=info from the Info link in the ro directory
4377         # works, and does not include the writecap URI.
4378         return d
4379
4380     def test_immutable_unknown(self):
4381         return self.test_unknown(immutable=True)
4382
4383     def test_mutant_dirnodes_are_omitted(self):
4384         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4385
4386         self.set_up_grid()
4387         c = self.g.clients[0]
4388         nm = c.nodemaker
4389         self.uris = {}
4390         self.fileurls = {}
4391
4392         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4393         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4394         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4395
4396         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4397         # test the dirnode and web layers separately.
4398
4399         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4400         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4401         # When the directory is read, the mutants should be silently disposed of, leaving
4402         # their lonely sibling.
4403         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4404         # because immutable directories don't have a writecap and therefore that field
4405         # isn't (and can't be) decrypted.
4406         # TODO: The field still exists in the netstring. Technically we should check what
4407         # happens if something is put there (_unpack_contents should raise ValueError),
4408         # but that can wait.
4409
4410         lonely_child = nm.create_from_cap(lonely_uri)
4411         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4412         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4413
4414         def _by_hook_or_by_crook():
4415             return True
4416         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4417             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4418
4419         mutant_write_in_ro_child.get_write_uri    = lambda: None
4420         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4421
4422         kids = {u"lonely":      (lonely_child, {}),
4423                 u"ro":          (mutant_ro_child, {}),
4424                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4425                 }
4426         d = c.create_immutable_dirnode(kids)
4427
4428         def _created(dn):
4429             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4430             self.failIf(dn.is_mutable())
4431             self.failUnless(dn.is_readonly())
4432             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4433             self.failIf(hasattr(dn._node, 'get_writekey'))
4434             rep = str(dn)
4435             self.failUnless("RO-IMM" in rep)
4436             cap = dn.get_cap()
4437             self.failUnlessIn("CHK", cap.to_string())
4438             self.cap = cap
4439             self.rootnode = dn
4440             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4441             return download_to_data(dn._node)
4442         d.addCallback(_created)
4443
4444         def _check_data(data):
4445             # Decode the netstring representation of the directory to check that all children
4446             # are present. This is a bit of an abstraction violation, but there's not really
4447             # any other way to do it given that the real DirectoryNode._unpack_contents would
4448             # strip the mutant children out (which is what we're trying to test, later).
4449             position = 0
4450             numkids = 0
4451             while position < len(data):
4452                 entries, position = split_netstring(data, 1, position)
4453                 entry = entries[0]
4454                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4455                 name = name_utf8.decode("utf-8")
4456                 self.failUnless(rwcapdata == "")
4457                 self.failUnless(name in kids)
4458                 (expected_child, ign) = kids[name]
4459                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4460                 numkids += 1
4461
4462             self.failUnlessReallyEqual(numkids, 3)
4463             return self.rootnode.list()
4464         d.addCallback(_check_data)
4465
4466         # Now when we use the real directory listing code, the mutants should be absent.
4467         def _check_kids(children):
4468             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4469             lonely_node, lonely_metadata = children[u"lonely"]
4470
4471             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4472             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4473         d.addCallback(_check_kids)
4474
4475         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4476         d.addCallback(lambda n: n.list())
4477         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4478
4479         # Make sure the lonely child can be listed in HTML...
4480         d.addCallback(lambda ign: self.GET(self.rooturl))
4481         def _check_html(res):
4482             self.failIfIn("URI:SSK", res)
4483             get_lonely = "".join([r'<td>FILE</td>',
4484                                   r'\s+<td>',
4485                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4486                                   r'</td>',
4487                                   r'\s+<td align="right">%d</td>' % len("one"),
4488                                   ])
4489             self.failUnless(re.search(get_lonely, res), res)
4490
4491             # find the More Info link for name, should be relative
4492             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4493             info_url = mo.group(1)
4494             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4495         d.addCallback(_check_html)
4496
4497         # ... and in JSON.
4498         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4499         def _check_json(res):
4500             data = simplejson.loads(res)
4501             self.failUnlessEqual(data[0], "dirnode")
4502             listed_children = data[1]["children"]
4503             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4504             ll_type, ll_data = listed_children[u"lonely"]
4505             self.failUnlessEqual(ll_type, "filenode")
4506             self.failIf("rw_uri" in ll_data)
4507             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4508         d.addCallback(_check_json)
4509         return d
4510
4511     def test_deep_check(self):
4512         self.basedir = "web/Grid/deep_check"
4513         self.set_up_grid()
4514         c0 = self.g.clients[0]
4515         self.uris = {}
4516         self.fileurls = {}
4517         DATA = "data" * 100
4518         d = c0.create_dirnode()
4519         def _stash_root_and_create_file(n):
4520             self.rootnode = n
4521             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4522             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4523         d.addCallback(_stash_root_and_create_file)
4524         def _stash_uri(fn, which):
4525             self.uris[which] = fn.get_uri()
4526             return fn
4527         d.addCallback(_stash_uri, "good")
4528         d.addCallback(lambda ign:
4529                       self.rootnode.add_file(u"small",
4530                                              upload.Data("literal",
4531                                                         convergence="")))
4532         d.addCallback(_stash_uri, "small")
4533         d.addCallback(lambda ign:
4534                       self.rootnode.add_file(u"sick",
4535                                              upload.Data(DATA+"1",
4536                                                         convergence="")))
4537         d.addCallback(_stash_uri, "sick")
4538
4539         # this tests that deep-check and stream-manifest will ignore
4540         # UnknownNode instances. Hopefully this will also cover deep-stats.
4541         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4542         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4543
4544         def _clobber_shares(ignored):
4545             self.delete_shares_numbered(self.uris["sick"], [0,1])
4546         d.addCallback(_clobber_shares)
4547
4548         # root
4549         # root/good
4550         # root/small
4551         # root/sick
4552         # root/future
4553
4554         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4555         def _done(res):
4556             try:
4557                 units = [simplejson.loads(line)
4558                          for line in res.splitlines()
4559                          if line]
4560             except ValueError:
4561                 print "response is:", res
4562                 print "undecodeable line was '%s'" % line
4563                 raise
4564             self.failUnlessReallyEqual(len(units), 5+1)
4565             # should be parent-first
4566             u0 = units[0]
4567             self.failUnlessEqual(u0["path"], [])
4568             self.failUnlessEqual(u0["type"], "directory")
4569             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4570             u0cr = u0["check-results"]
4571             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4572
4573             ugood = [u for u in units
4574                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4575             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4576             ugoodcr = ugood["check-results"]
4577             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4578
4579             stats = units[-1]
4580             self.failUnlessEqual(stats["type"], "stats")
4581             s = stats["stats"]
4582             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4583             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4584             self.failUnlessReallyEqual(s["count-directories"], 1)
4585             self.failUnlessReallyEqual(s["count-unknown"], 1)
4586         d.addCallback(_done)
4587
4588         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4589         def _check_manifest(res):
4590             self.failUnless(res.endswith("\n"))
4591             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4592             self.failUnlessReallyEqual(len(units), 5+1)
4593             self.failUnlessEqual(units[-1]["type"], "stats")
4594             first = units[0]
4595             self.failUnlessEqual(first["path"], [])
4596             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4597             self.failUnlessEqual(first["type"], "directory")
4598             stats = units[-1]["stats"]
4599             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4600             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4601             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4602             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4603             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4604         d.addCallback(_check_manifest)
4605
4606         # now add root/subdir and root/subdir/grandchild, then make subdir
4607         # unrecoverable, then see what happens
4608
4609         d.addCallback(lambda ign:
4610                       self.rootnode.create_subdirectory(u"subdir"))
4611         d.addCallback(_stash_uri, "subdir")
4612         d.addCallback(lambda subdir_node:
4613                       subdir_node.add_file(u"grandchild",
4614                                            upload.Data(DATA+"2",
4615                                                        convergence="")))
4616         d.addCallback(_stash_uri, "grandchild")
4617
4618         d.addCallback(lambda ign:
4619                       self.delete_shares_numbered(self.uris["subdir"],
4620                                                   range(1, 10)))
4621
4622         # root
4623         # root/good
4624         # root/small
4625         # root/sick
4626         # root/future
4627         # root/subdir [unrecoverable]
4628         # root/subdir/grandchild
4629
4630         # how should a streaming-JSON API indicate fatal error?
4631         # answer: emit ERROR: instead of a JSON string
4632
4633         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4634         def _check_broken_manifest(res):
4635             lines = res.splitlines()
4636             error_lines = [i
4637                            for (i,line) in enumerate(lines)
4638                            if line.startswith("ERROR:")]
4639             if not error_lines:
4640                 self.fail("no ERROR: in output: %s" % (res,))
4641             first_error = error_lines[0]
4642             error_line = lines[first_error]
4643             error_msg = lines[first_error+1:]
4644             error_msg_s = "\n".join(error_msg) + "\n"
4645             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4646                               error_line)
4647             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4648             units = [simplejson.loads(line) for line in lines[:first_error]]
4649             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4650             last_unit = units[-1]
4651             self.failUnlessEqual(last_unit["path"], ["subdir"])
4652         d.addCallback(_check_broken_manifest)
4653
4654         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4655         def _check_broken_deepcheck(res):
4656             lines = res.splitlines()
4657             error_lines = [i
4658                            for (i,line) in enumerate(lines)
4659                            if line.startswith("ERROR:")]
4660             if not error_lines:
4661                 self.fail("no ERROR: in output: %s" % (res,))
4662             first_error = error_lines[0]
4663             error_line = lines[first_error]
4664             error_msg = lines[first_error+1:]
4665             error_msg_s = "\n".join(error_msg) + "\n"
4666             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4667                               error_line)
4668             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4669             units = [simplejson.loads(line) for line in lines[:first_error]]
4670             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4671             last_unit = units[-1]
4672             self.failUnlessEqual(last_unit["path"], ["subdir"])
4673             r = last_unit["check-results"]["results"]
4674             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4675             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4676             self.failUnlessReallyEqual(r["recoverable"], False)
4677         d.addCallback(_check_broken_deepcheck)
4678
4679         d.addErrback(self.explain_web_error)
4680         return d
4681
4682     def test_deep_check_and_repair(self):
4683         self.basedir = "web/Grid/deep_check_and_repair"
4684         self.set_up_grid()
4685         c0 = self.g.clients[0]
4686         self.uris = {}
4687         self.fileurls = {}
4688         DATA = "data" * 100
4689         d = c0.create_dirnode()
4690         def _stash_root_and_create_file(n):
4691             self.rootnode = n
4692             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4693             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4694         d.addCallback(_stash_root_and_create_file)
4695         def _stash_uri(fn, which):
4696             self.uris[which] = fn.get_uri()
4697         d.addCallback(_stash_uri, "good")
4698         d.addCallback(lambda ign:
4699                       self.rootnode.add_file(u"small",
4700                                              upload.Data("literal",
4701                                                         convergence="")))
4702         d.addCallback(_stash_uri, "small")
4703         d.addCallback(lambda ign:
4704                       self.rootnode.add_file(u"sick",
4705                                              upload.Data(DATA+"1",
4706                                                         convergence="")))
4707         d.addCallback(_stash_uri, "sick")
4708         #d.addCallback(lambda ign:
4709         #              self.rootnode.add_file(u"dead",
4710         #                                     upload.Data(DATA+"2",
4711         #                                                convergence="")))
4712         #d.addCallback(_stash_uri, "dead")
4713
4714         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4715         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4716         #d.addCallback(_stash_uri, "corrupt")
4717
4718         def _clobber_shares(ignored):
4719             good_shares = self.find_uri_shares(self.uris["good"])
4720             self.failUnlessReallyEqual(len(good_shares), 10)
4721             sick_shares = self.find_uri_shares(self.uris["sick"])
4722             os.unlink(sick_shares[0][2])
4723             #dead_shares = self.find_uri_shares(self.uris["dead"])
4724             #for i in range(1, 10):
4725             #    os.unlink(dead_shares[i][2])
4726
4727             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4728             #cso = CorruptShareOptions()
4729             #cso.stdout = StringIO()
4730             #cso.parseOptions([c_shares[0][2]])
4731             #corrupt_share(cso)
4732         d.addCallback(_clobber_shares)
4733
4734         # root
4735         # root/good   CHK, 10 shares
4736         # root/small  LIT
4737         # root/sick   CHK, 9 shares
4738
4739         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4740         def _done(res):
4741             units = [simplejson.loads(line)
4742                      for line in res.splitlines()
4743                      if line]
4744             self.failUnlessReallyEqual(len(units), 4+1)
4745             # should be parent-first
4746             u0 = units[0]
4747             self.failUnlessEqual(u0["path"], [])
4748             self.failUnlessEqual(u0["type"], "directory")
4749             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4750             u0crr = u0["check-and-repair-results"]
4751             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4752             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4753
4754             ugood = [u for u in units
4755                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4756             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4757             ugoodcrr = ugood["check-and-repair-results"]
4758             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4759             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4760
4761             usick = [u for u in units
4762                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4763             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4764             usickcrr = usick["check-and-repair-results"]
4765             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4766             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4767             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4768             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4769
4770             stats = units[-1]
4771             self.failUnlessEqual(stats["type"], "stats")
4772             s = stats["stats"]
4773             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4774             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4775             self.failUnlessReallyEqual(s["count-directories"], 1)
4776         d.addCallback(_done)
4777
4778         d.addErrback(self.explain_web_error)
4779         return d
4780
4781     def _count_leases(self, ignored, which):
4782         u = self.uris[which]
4783         shares = self.find_uri_shares(u)
4784         lease_counts = []
4785         for shnum, serverid, fn in shares:
4786             sf = get_share_file(fn)
4787             num_leases = len(list(sf.get_leases()))
4788             lease_counts.append( (fn, num_leases) )
4789         return lease_counts
4790
4791     def _assert_leasecount(self, lease_counts, expected):
4792         for (fn, num_leases) in lease_counts:
4793             if num_leases != expected:
4794                 self.fail("expected %d leases, have %d, on %s" %
4795                           (expected, num_leases, fn))
4796
4797     def test_add_lease(self):
4798         self.basedir = "web/Grid/add_lease"
4799         self.set_up_grid(num_clients=2)
4800         c0 = self.g.clients[0]
4801         self.uris = {}
4802         DATA = "data" * 100
4803         d = c0.upload(upload.Data(DATA, convergence=""))
4804         def _stash_uri(ur, which):
4805             self.uris[which] = ur.uri
4806         d.addCallback(_stash_uri, "one")
4807         d.addCallback(lambda ign:
4808                       c0.upload(upload.Data(DATA+"1", convergence="")))
4809         d.addCallback(_stash_uri, "two")
4810         def _stash_mutable_uri(n, which):
4811             self.uris[which] = n.get_uri()
4812             assert isinstance(self.uris[which], str)
4813         d.addCallback(lambda ign:
4814             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4815         d.addCallback(_stash_mutable_uri, "mutable")
4816
4817         def _compute_fileurls(ignored):
4818             self.fileurls = {}
4819             for which in self.uris:
4820                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4821         d.addCallback(_compute_fileurls)
4822
4823         d.addCallback(self._count_leases, "one")
4824         d.addCallback(self._assert_leasecount, 1)
4825         d.addCallback(self._count_leases, "two")
4826         d.addCallback(self._assert_leasecount, 1)
4827         d.addCallback(self._count_leases, "mutable")
4828         d.addCallback(self._assert_leasecount, 1)
4829
4830         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4831         def _got_html_good(res):
4832             self.failUnless("Healthy" in res, res)
4833             self.failIf("Not Healthy" in res, res)
4834         d.addCallback(_got_html_good)
4835
4836         d.addCallback(self._count_leases, "one")
4837         d.addCallback(self._assert_leasecount, 1)
4838         d.addCallback(self._count_leases, "two")
4839         d.addCallback(self._assert_leasecount, 1)
4840         d.addCallback(self._count_leases, "mutable")
4841         d.addCallback(self._assert_leasecount, 1)
4842
4843         # this CHECK uses the original client, which uses the same
4844         # lease-secrets, so it will just renew the original lease
4845         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4846         d.addCallback(_got_html_good)
4847
4848         d.addCallback(self._count_leases, "one")
4849         d.addCallback(self._assert_leasecount, 1)
4850         d.addCallback(self._count_leases, "two")
4851         d.addCallback(self._assert_leasecount, 1)
4852         d.addCallback(self._count_leases, "mutable")
4853         d.addCallback(self._assert_leasecount, 1)
4854
4855         # this CHECK uses an alternate client, which adds a second lease
4856         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4857         d.addCallback(_got_html_good)
4858
4859         d.addCallback(self._count_leases, "one")
4860         d.addCallback(self._assert_leasecount, 2)
4861         d.addCallback(self._count_leases, "two")
4862         d.addCallback(self._assert_leasecount, 1)
4863         d.addCallback(self._count_leases, "mutable")
4864         d.addCallback(self._assert_leasecount, 1)
4865
4866         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4867         d.addCallback(_got_html_good)
4868
4869         d.addCallback(self._count_leases, "one")
4870         d.addCallback(self._assert_leasecount, 2)
4871         d.addCallback(self._count_leases, "two")
4872         d.addCallback(self._assert_leasecount, 1)
4873         d.addCallback(self._count_leases, "mutable")
4874         d.addCallback(self._assert_leasecount, 1)
4875
4876         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4877                       clientnum=1)
4878         d.addCallback(_got_html_good)
4879
4880         d.addCallback(self._count_leases, "one")
4881         d.addCallback(self._assert_leasecount, 2)
4882         d.addCallback(self._count_leases, "two")
4883         d.addCallback(self._assert_leasecount, 1)
4884         d.addCallback(self._count_leases, "mutable")
4885         d.addCallback(self._assert_leasecount, 2)
4886
4887         d.addErrback(self.explain_web_error)
4888         return d
4889
4890     def test_deep_add_lease(self):
4891         self.basedir = "web/Grid/deep_add_lease"
4892         self.set_up_grid(num_clients=2)
4893         c0 = self.g.clients[0]
4894         self.uris = {}
4895         self.fileurls = {}
4896         DATA = "data" * 100
4897         d = c0.create_dirnode()
4898         def _stash_root_and_create_file(n):
4899             self.rootnode = n
4900             self.uris["root"] = n.get_uri()
4901             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4902             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4903         d.addCallback(_stash_root_and_create_file)
4904         def _stash_uri(fn, which):
4905             self.uris[which] = fn.get_uri()
4906         d.addCallback(_stash_uri, "one")
4907         d.addCallback(lambda ign:
4908                       self.rootnode.add_file(u"small",
4909                                              upload.Data("literal",
4910                                                         convergence="")))
4911         d.addCallback(_stash_uri, "small")
4912
4913         d.addCallback(lambda ign:
4914             c0.create_mutable_file(publish.MutableData("mutable")))
4915         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4916         d.addCallback(_stash_uri, "mutable")
4917
4918         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4919         def _done(res):
4920             units = [simplejson.loads(line)
4921                      for line in res.splitlines()
4922                      if line]
4923             # root, one, small, mutable,   stats
4924             self.failUnlessReallyEqual(len(units), 4+1)
4925         d.addCallback(_done)
4926
4927         d.addCallback(self._count_leases, "root")
4928         d.addCallback(self._assert_leasecount, 1)
4929         d.addCallback(self._count_leases, "one")
4930         d.addCallback(self._assert_leasecount, 1)
4931         d.addCallback(self._count_leases, "mutable")
4932         d.addCallback(self._assert_leasecount, 1)
4933
4934         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4935         d.addCallback(_done)
4936
4937         d.addCallback(self._count_leases, "root")
4938         d.addCallback(self._assert_leasecount, 1)
4939         d.addCallback(self._count_leases, "one")
4940         d.addCallback(self._assert_leasecount, 1)
4941         d.addCallback(self._count_leases, "mutable")
4942         d.addCallback(self._assert_leasecount, 1)
4943
4944         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4945                       clientnum=1)
4946         d.addCallback(_done)
4947
4948         d.addCallback(self._count_leases, "root")
4949         d.addCallback(self._assert_leasecount, 2)
4950         d.addCallback(self._count_leases, "one")
4951         d.addCallback(self._assert_leasecount, 2)
4952         d.addCallback(self._count_leases, "mutable")
4953         d.addCallback(self._assert_leasecount, 2)
4954
4955         d.addErrback(self.explain_web_error)
4956         return d
4957
4958
4959     def test_exceptions(self):
4960         self.basedir = "web/Grid/exceptions"
4961         self.set_up_grid(num_clients=1, num_servers=2)
4962         c0 = self.g.clients[0]
4963         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4964         self.fileurls = {}
4965         DATA = "data" * 100
4966         d = c0.create_dirnode()
4967         def _stash_root(n):
4968             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4969             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4970             return n
4971         d.addCallback(_stash_root)
4972         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4973         def _stash_bad(ur):
4974             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4975             self.delete_shares_numbered(ur.uri, range(1,10))
4976
4977             u = uri.from_string(ur.uri)
4978             u.key = testutil.flip_bit(u.key, 0)
4979             baduri = u.to_string()
4980             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4981         d.addCallback(_stash_bad)
4982         d.addCallback(lambda ign: c0.create_dirnode())
4983         def _mangle_dirnode_1share(n):
4984             u = n.get_uri()
4985             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4986             self.fileurls["dir-1share-json"] = url + "?t=json"
4987             self.delete_shares_numbered(u, range(1,10))
4988         d.addCallback(_mangle_dirnode_1share)
4989         d.addCallback(lambda ign: c0.create_dirnode())
4990         def _mangle_dirnode_0share(n):
4991             u = n.get_uri()
4992             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4993             self.fileurls["dir-0share-json"] = url + "?t=json"
4994             self.delete_shares_numbered(u, range(0,10))
4995         d.addCallback(_mangle_dirnode_0share)
4996
4997         # NotEnoughSharesError should be reported sensibly, with a
4998         # text/plain explanation of the problem, and perhaps some
4999         # information on which shares *could* be found.
5000
5001         d.addCallback(lambda ignored:
5002                       self.shouldHTTPError("GET unrecoverable",
5003                                            410, "Gone", "NoSharesError",
5004                                            self.GET, self.fileurls["0shares"]))
5005         def _check_zero_shares(body):
5006             self.failIf("<html>" in body, body)
5007             body = " ".join(body.strip().split())
5008             exp = ("NoSharesError: no shares could be found. "
5009                    "Zero shares usually indicates a corrupt URI, or that "
5010                    "no servers were connected, but it might also indicate "
5011                    "severe corruption. You should perform a filecheck on "
5012                    "this object to learn more. The full error message is: "
5013                    "no shares (need 3). Last failure: None")
5014             self.failUnlessReallyEqual(exp, body)
5015         d.addCallback(_check_zero_shares)
5016
5017
5018         d.addCallback(lambda ignored:
5019                       self.shouldHTTPError("GET 1share",
5020                                            410, "Gone", "NotEnoughSharesError",
5021                                            self.GET, self.fileurls["1share"]))
5022         def _check_one_share(body):
5023             self.failIf("<html>" in body, body)
5024             body = " ".join(body.strip().split())
5025             msgbase = ("NotEnoughSharesError: This indicates that some "
5026                        "servers were unavailable, or that shares have been "
5027                        "lost to server departure, hard drive failure, or disk "
5028                        "corruption. You should perform a filecheck on "
5029                        "this object to learn more. The full error message is:"
5030                        )
5031             msg1 = msgbase + (" ran out of shares:"
5032                               " complete=sh0"
5033                               " pending="
5034                               " overdue= unused= need 3. Last failure: None")
5035             msg2 = msgbase + (" ran out of shares:"
5036                               " complete="
5037                               " pending=Share(sh0-on-xgru5)"
5038                               " overdue= unused= need 3. Last failure: None")
5039             self.failUnless(body == msg1 or body == msg2, body)
5040         d.addCallback(_check_one_share)
5041
5042         d.addCallback(lambda ignored:
5043                       self.shouldHTTPError("GET imaginary",
5044                                            404, "Not Found", None,
5045                                            self.GET, self.fileurls["imaginary"]))
5046         def _missing_child(body):
5047             self.failUnless("No such child: imaginary" in body, body)
5048         d.addCallback(_missing_child)
5049
5050         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5051         def _check_0shares_dir_html(body):
5052             self.failUnless("<html>" in body, body)
5053             # we should see the regular page, but without the child table or
5054             # the dirops forms
5055             body = " ".join(body.strip().split())
5056             self.failUnlessIn('href="?t=info">More info on this directory',
5057                               body)
5058             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5059                    "could not be retrieved, because there were insufficient "
5060                    "good shares. This might indicate that no servers were "
5061                    "connected, insufficient servers were connected, the URI "
5062                    "was corrupt, or that shares have been lost due to server "
5063                    "departure, hard drive failure, or disk corruption. You "
5064                    "should perform a filecheck on this object to learn more.")
5065             self.failUnlessIn(exp, body)
5066             self.failUnlessIn("No upload forms: directory is unreadable", body)
5067         d.addCallback(_check_0shares_dir_html)
5068
5069         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5070         def _check_1shares_dir_html(body):
5071             # at some point, we'll split UnrecoverableFileError into 0-shares
5072             # and some-shares like we did for immutable files (since there
5073             # are different sorts of advice to offer in each case). For now,
5074             # they present the same way.
5075             self.failUnless("<html>" in body, body)
5076             body = " ".join(body.strip().split())
5077             self.failUnlessIn('href="?t=info">More info on this directory',
5078                               body)
5079             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5080                    "could not be retrieved, because there were insufficient "
5081                    "good shares. This might indicate that no servers were "
5082                    "connected, insufficient servers were connected, the URI "
5083                    "was corrupt, or that shares have been lost due to server "
5084                    "departure, hard drive failure, or disk corruption. You "
5085                    "should perform a filecheck on this object to learn more.")
5086             self.failUnlessIn(exp, body)
5087             self.failUnlessIn("No upload forms: directory is unreadable", body)
5088         d.addCallback(_check_1shares_dir_html)
5089
5090         d.addCallback(lambda ignored:
5091                       self.shouldHTTPError("GET dir-0share-json",
5092                                            410, "Gone", "UnrecoverableFileError",
5093                                            self.GET,
5094                                            self.fileurls["dir-0share-json"]))
5095         def _check_unrecoverable_file(body):
5096             self.failIf("<html>" in body, body)
5097             body = " ".join(body.strip().split())
5098             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5099                    "could not be retrieved, because there were insufficient "
5100                    "good shares. This might indicate that no servers were "
5101                    "connected, insufficient servers were connected, the URI "
5102                    "was corrupt, or that shares have been lost due to server "
5103                    "departure, hard drive failure, or disk corruption. You "
5104                    "should perform a filecheck on this object to learn more.")
5105             self.failUnlessReallyEqual(exp, body)
5106         d.addCallback(_check_unrecoverable_file)
5107
5108         d.addCallback(lambda ignored:
5109                       self.shouldHTTPError("GET dir-1share-json",
5110                                            410, "Gone", "UnrecoverableFileError",
5111                                            self.GET,
5112                                            self.fileurls["dir-1share-json"]))
5113         d.addCallback(_check_unrecoverable_file)
5114
5115         d.addCallback(lambda ignored:
5116                       self.shouldHTTPError("GET imaginary",
5117                                            404, "Not Found", None,
5118                                            self.GET, self.fileurls["imaginary"]))
5119
5120         # attach a webapi child that throws a random error, to test how it
5121         # gets rendered.
5122         w = c0.getServiceNamed("webish")
5123         w.root.putChild("ERRORBOOM", ErrorBoom())
5124
5125         # "Accept: */*" :        should get a text/html stack trace
5126         # "Accept: text/plain" : should get a text/plain stack trace
5127         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5128         # no Accept header:      should get a text/html stack trace
5129
5130         d.addCallback(lambda ignored:
5131                       self.shouldHTTPError("GET errorboom_html",
5132                                            500, "Internal Server Error", None,
5133                                            self.GET, "ERRORBOOM",
5134                                            headers={"accept": ["*/*"]}))
5135         def _internal_error_html1(body):
5136             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5137         d.addCallback(_internal_error_html1)
5138
5139         d.addCallback(lambda ignored:
5140                       self.shouldHTTPError("GET errorboom_text",
5141                                            500, "Internal Server Error", None,
5142                                            self.GET, "ERRORBOOM",
5143                                            headers={"accept": ["text/plain"]}))
5144         def _internal_error_text2(body):
5145             self.failIf("<html>" in body, body)
5146             self.failUnless(body.startswith("Traceback "), body)
5147         d.addCallback(_internal_error_text2)
5148
5149         CLI_accepts = "text/plain, application/octet-stream"
5150         d.addCallback(lambda ignored:
5151                       self.shouldHTTPError("GET errorboom_text",
5152                                            500, "Internal Server Error", None,
5153                                            self.GET, "ERRORBOOM",
5154                                            headers={"accept": [CLI_accepts]}))
5155         def _internal_error_text3(body):
5156             self.failIf("<html>" in body, body)
5157             self.failUnless(body.startswith("Traceback "), body)
5158         d.addCallback(_internal_error_text3)
5159
5160         d.addCallback(lambda ignored:
5161                       self.shouldHTTPError("GET errorboom_text",
5162                                            500, "Internal Server Error", None,
5163                                            self.GET, "ERRORBOOM"))
5164         def _internal_error_html4(body):
5165             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5166         d.addCallback(_internal_error_html4)
5167
5168         def _flush_errors(res):
5169             # Trial: please ignore the CompletelyUnhandledError in the logs
5170             self.flushLoggedErrors(CompletelyUnhandledError)
5171             return res
5172         d.addBoth(_flush_errors)
5173
5174         return d
5175
5176     def test_blacklist(self):
5177         # download from a blacklisted URI, get an error
5178         self.basedir = "web/Grid/blacklist"
5179         self.set_up_grid()
5180         c0 = self.g.clients[0]
5181         c0_basedir = c0.basedir
5182         fn = os.path.join(c0_basedir, "access.blacklist")
5183         self.uris = {}
5184         DATA = "off-limits " * 50
5185
5186         d = c0.upload(upload.Data(DATA, convergence=""))
5187         def _stash_uri_and_create_dir(ur):
5188             self.uri = ur.uri
5189             self.url = "uri/"+self.uri
5190             u = uri.from_string_filenode(self.uri)
5191             self.si = u.get_storage_index()
5192             childnode = c0.create_node_from_uri(self.uri, None)
5193             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5194         d.addCallback(_stash_uri_and_create_dir)
5195         def _stash_dir(node):
5196             self.dir_node = node
5197             self.dir_uri = node.get_uri()
5198             self.dir_url = "uri/"+self.dir_uri
5199         d.addCallback(_stash_dir)
5200         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5201         def _check_dir_html(body):
5202             self.failUnlessIn("<html>", body)
5203             self.failUnlessIn("blacklisted.txt</a>", body)
5204         d.addCallback(_check_dir_html)
5205         d.addCallback(lambda ign: self.GET(self.url))
5206         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5207
5208         def _blacklist(ign):
5209             f = open(fn, "w")
5210             f.write(" # this is a comment\n")
5211             f.write(" \n")
5212             f.write("\n") # also exercise blank lines
5213             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5214             f.close()
5215             # clients should be checking the blacklist each time, so we don't
5216             # need to restart the client
5217         d.addCallback(_blacklist)
5218         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5219                                                        403, "Forbidden",
5220                                                        "Access Prohibited: off-limits",
5221                                                        self.GET, self.url))
5222
5223         # We should still be able to list the parent directory, in HTML...
5224         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5225         def _check_dir_html2(body):
5226             self.failUnlessIn("<html>", body)
5227             self.failUnlessIn("blacklisted.txt</strike>", body)
5228         d.addCallback(_check_dir_html2)
5229
5230         # ... and in JSON (used by CLI).
5231         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5232         def _check_dir_json(res):
5233             data = simplejson.loads(res)
5234             self.failUnless(isinstance(data, list), data)
5235             self.failUnlessEqual(data[0], "dirnode")
5236             self.failUnless(isinstance(data[1], dict), data)
5237             self.failUnlessIn("children", data[1])
5238             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5239             childdata = data[1]["children"]["blacklisted.txt"]
5240             self.failUnless(isinstance(childdata, list), data)
5241             self.failUnlessEqual(childdata[0], "filenode")
5242             self.failUnless(isinstance(childdata[1], dict), data)
5243         d.addCallback(_check_dir_json)
5244
5245         def _unblacklist(ign):
5246             open(fn, "w").close()
5247             # the Blacklist object watches mtime to tell when the file has
5248             # changed, but on windows this test will run faster than the
5249             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5250             # to force a reload.
5251             self.g.clients[0].blacklist.last_mtime -= 2.0
5252         d.addCallback(_unblacklist)
5253
5254         # now a read should work
5255         d.addCallback(lambda ign: self.GET(self.url))
5256         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5257
5258         # read again to exercise the blacklist-is-unchanged logic
5259         d.addCallback(lambda ign: self.GET(self.url))
5260         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5261
5262         # now add a blacklisted directory, and make sure files under it are
5263         # refused too
5264         def _add_dir(ign):
5265             childnode = c0.create_node_from_uri(self.uri, None)
5266             return c0.create_dirnode({u"child": (childnode,{}) })
5267         d.addCallback(_add_dir)
5268         def _get_dircap(dn):
5269             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5270             self.dir_url_base = "uri/"+dn.get_write_uri()
5271             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5272             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5273             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5274             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5275         d.addCallback(_get_dircap)
5276         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5277         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5278         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5279         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5280         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5281         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5282         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5283         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5284         d.addCallback(lambda ign: self.GET(self.child_url))
5285         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5286
5287         def _block_dir(ign):
5288             f = open(fn, "w")
5289             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5290             f.close()
5291             self.g.clients[0].blacklist.last_mtime -= 2.0
5292         d.addCallback(_block_dir)
5293         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5294                                                        403, "Forbidden",
5295                                                        "Access Prohibited: dir-off-limits",
5296                                                        self.GET, self.dir_url_base))
5297         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5298                                                        403, "Forbidden",
5299                                                        "Access Prohibited: dir-off-limits",
5300                                                        self.GET, self.dir_url_json1))
5301         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5302                                                        403, "Forbidden",
5303                                                        "Access Prohibited: dir-off-limits",
5304                                                        self.GET, self.dir_url_json2))
5305         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5306                                                        403, "Forbidden",
5307                                                        "Access Prohibited: dir-off-limits",
5308                                                        self.GET, self.dir_url_json_ro))
5309         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5310                                                        403, "Forbidden",
5311                                                        "Access Prohibited: dir-off-limits",
5312                                                        self.GET, self.child_url))
5313         return d
5314
5315
5316 class CompletelyUnhandledError(Exception):
5317     pass
5318 class ErrorBoom(rend.Page):
5319     def beforeRender(self, ctx):
5320         raise CompletelyUnhandledError("whoops")