src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649         d.addCallback(_check_dl_json)
 650         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 651         def _check_ul(res):
 652             self.failUnless("File Upload Status" in res, res)
 653         d.addCallback(_check_ul)
 654         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 655         def _check_mapupdate(res):
 656             self.failUnless("Mutable File Servermap Update Status" in res, res)
 657         d.addCallback(_check_mapupdate)
 658         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 659         def _check_publish(res):
 660             self.failUnless("Mutable File Publish Status" in res, res)
 661         d.addCallback(_check_publish)
 662         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 663         def _check_retrieve(res):
 664             self.failUnless("Mutable File Retrieve Status" in res, res)
 665         d.addCallback(_check_retrieve)
 666
 667         return d
 668
 669     def test_status_numbers(self):
 670         drrm = status.DownloadResultsRendererMixin()
 671         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 680
 681         urrm = status.UploadResultsRendererMixin()
 682         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 691
 692     def test_GET_FILEURL(self):
 693         d = self.GET(self.public_url + "/foo/bar.txt")
 694         d.addCallback(self.failUnlessIsBarDotTxt)
 695         return d
 696
 697     def test_GET_FILEURL_range(self):
 698         headers = {"range": "bytes=1-10"}
 699         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 700                      return_response=True)
 701         def _got((res, status, headers)):
 702             self.failUnlessReallyEqual(int(status), 206)
 703             self.failUnless(headers.has_key("content-range"))
 704             self.failUnlessReallyEqual(headers["content-range"][0],
 705                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 706             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 707         d.addCallback(_got)
 708         return d
 709
 710     def test_GET_FILEURL_partial_range(self):
 711         headers = {"range": "bytes=5-"}
 712         length  = len(self.BAR_CONTENTS)
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 206)
 717             self.failUnless(headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(headers["content-range"][0],
 719                                        "bytes 5-%d/%d" % (length-1, length))
 720             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 721         d.addCallback(_got)
 722         return d
 723
 724     def test_GET_FILEURL_partial_end_range(self):
 725         headers = {"range": "bytes=-5"}
 726         length  = len(self.BAR_CONTENTS)
 727         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 728                      return_response=True)
 729         def _got((res, status, headers)):
 730             self.failUnlessReallyEqual(int(status), 206)
 731             self.failUnless(headers.has_key("content-range"))
 732             self.failUnlessReallyEqual(headers["content-range"][0],
 733                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 734             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_partial_range_overrun(self):
 739         headers = {"range": "bytes=100-200"}
 740         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 741                              "416 Requested Range not satisfiable",
 742                              "First beyond end of file",
 743                              self.GET, self.public_url + "/foo/bar.txt",
 744                              headers=headers)
 745         return d
 746
 747     def test_HEAD_FILEURL_range(self):
 748         headers = {"range": "bytes=1-10"}
 749         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 750                      return_response=True)
 751         def _got((res, status, headers)):
 752             self.failUnlessReallyEqual(res, "")
 753             self.failUnlessReallyEqual(int(status), 206)
 754             self.failUnless(headers.has_key("content-range"))
 755             self.failUnlessReallyEqual(headers["content-range"][0],
 756                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_HEAD_FILEURL_partial_range(self):
 761         headers = {"range": "bytes=5-"}
 762         length  = len(self.BAR_CONTENTS)
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 764                      return_response=True)
 765         def _got((res, status, headers)):
 766             self.failUnlessReallyEqual(int(status), 206)
 767             self.failUnless(headers.has_key("content-range"))
 768             self.failUnlessReallyEqual(headers["content-range"][0],
 769                                        "bytes 5-%d/%d" % (length-1, length))
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_HEAD_FILEURL_partial_end_range(self):
 774         headers = {"range": "bytes=-5"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range_overrun(self):
 787         headers = {"range": "bytes=100-200"}
 788         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 789                              "416 Requested Range not satisfiable",
 790                              "",
 791                              self.HEAD, self.public_url + "/foo/bar.txt",
 792                              headers=headers)
 793         return d
 794
 795     def test_GET_FILEURL_range_bad(self):
 796         headers = {"range": "BOGUS=fizbop-quarnak"}
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 200)
 801             self.failUnless(not headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 803         d.addCallback(_got)
 804         return d
 805
 806     def test_HEAD_FILEURL(self):
 807         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 808         def _got((res, status, headers)):
 809             self.failUnlessReallyEqual(res, "")
 810             self.failUnlessReallyEqual(headers["content-length"][0],
 811                                        str(len(self.BAR_CONTENTS)))
 812             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 813         d.addCallback(_got)
 814         return d
 815
 816     def test_GET_FILEURL_named(self):
 817         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 818         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 819         d = self.GET(base + "/@@name=/blah.txt")
 820         d.addCallback(self.failUnlessIsBarDotTxt)
 821         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 822         d.addCallback(self.failUnlessIsBarDotTxt)
 823         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 824         d.addCallback(self.failUnlessIsBarDotTxt)
 825         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 826         d.addCallback(self.failUnlessIsBarDotTxt)
 827         save_url = base + "?save=true&filename=blah.txt"
 828         d.addCallback(lambda res: self.GET(save_url))
 829         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 830         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 831         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 832         u_url = base + "?save=true&filename=" + u_fn_e
 833         d.addCallback(lambda res: self.GET(u_url))
 834         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 835         return d
 836
 837     def test_PUT_FILEURL_named_bad(self):
 838         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 839         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 840                              "400 Bad Request",
 841                              "/file can only be used with GET or HEAD",
 842                              self.PUT, base + "/@@name=/blah.txt", "")
 843         return d
 844
 845
 846     def test_GET_DIRURL_named_bad(self):
 847         base = "/file/%s" % urllib.quote(self._foo_uri)
 848         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 849                              "400 Bad Request",
 850                              "is not a file-cap",
 851                              self.GET, base + "/@@name=/blah.txt")
 852         return d
 853
 854     def test_GET_slash_file_bad(self):
 855         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 856                              "404 Not Found",
 857                              "/file must be followed by a file-cap and a name",
 858                              self.GET, "/file")
 859         return d
 860
 861     def test_GET_unhandled_URI_named(self):
 862         contents, n, newuri = self.makefile(12)
 863         verifier_cap = n.get_verify_cap().to_string()
 864         base = "/file/%s" % urllib.quote(verifier_cap)
 865         # client.create_node_from_uri() can't handle verify-caps
 866         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 867                              "400 Bad Request", "is not a file-cap",
 868                              self.GET, base)
 869         return d
 870
 871     def test_GET_unhandled_URI(self):
 872         contents, n, newuri = self.makefile(12)
 873         verifier_cap = n.get_verify_cap().to_string()
 874         base = "/uri/%s" % urllib.quote(verifier_cap)
 875         # client.create_node_from_uri() can't handle verify-caps
 876         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 877                              "400 Bad Request",
 878                              "GET unknown URI type: can only do t=info",
 879                              self.GET, base)
 880         return d
 881
 882     def test_GET_FILE_URI(self):
 883         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 884         d = self.GET(base)
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         return d
 887
 888     def test_GET_FILE_URI_mdmf(self):
 889         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 890         d = self.GET(base)
 891         d.addCallback(self.failUnlessIsQuuxDotTxt)
 892         return d
 893
 894     def test_GET_FILE_URI_mdmf_extensions(self):
 895         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 896         d = self.GET(base)
 897         d.addCallback(self.failUnlessIsQuuxDotTxt)
 898         return d
 899
 900     def test_GET_FILE_URI_mdmf_readonly(self):
 901         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 902         d = self.GET(base)
 903         d.addCallback(self.failUnlessIsQuuxDotTxt)
 904         return d
 905
 906     def test_GET_FILE_URI_badchild(self):
 907         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 908         errmsg = "Files have no children, certainly not named 'boguschild'"
 909         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 910                              "400 Bad Request", errmsg,
 911                              self.GET, base)
 912         return d
 913
 914     def test_PUT_FILE_URI_badchild(self):
 915         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 916         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 917         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 918                              "400 Bad Request", errmsg,
 919                              self.PUT, base, "")
 920         return d
 921
 922     def test_PUT_FILE_URI_mdmf(self):
 923         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 924         self._quux_new_contents = "new_contents"
 925         d = self.GET(base)
 926         d.addCallback(lambda res:
 927             self.failUnlessIsQuuxDotTxt(res))
 928         d.addCallback(lambda ignored:
 929             self.PUT(base, self._quux_new_contents))
 930         d.addCallback(lambda ignored:
 931             self.GET(base))
 932         d.addCallback(lambda res:
 933             self.failUnlessReallyEqual(res, self._quux_new_contents))
 934         return d
 935
 936     def test_PUT_FILE_URI_mdmf_extensions(self):
 937         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 938         self._quux_new_contents = "new_contents"
 939         d = self.GET(base)
 940         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 941         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 942         d.addCallback(lambda ignored: self.GET(base))
 943         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 944                                                        res))
 945         return d
 946
 947     def test_PUT_FILE_URI_mdmf_readonly(self):
 948         # We're not allowed to PUT things to a readonly cap.
 949         base = "/uri/%s" % self._quux_txt_readonly_uri
 950         d = self.GET(base)
 951         d.addCallback(lambda res:
 952             self.failUnlessIsQuuxDotTxt(res))
 953         # What should we get here? We get a 500 error now; that's not right.
 954         d.addCallback(lambda ignored:
 955             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 956                              "400 Bad Request", "read-only cap",
 957                              self.PUT, base, "new data"))
 958         return d
 959
 960     def test_PUT_FILE_URI_sdmf_readonly(self):
 961         # We're not allowed to put things to a readonly cap.
 962         base = "/uri/%s" % self._baz_txt_readonly_uri
 963         d = self.GET(base)
 964         d.addCallback(lambda res:
 965             self.failUnlessIsBazDotTxt(res))
 966         d.addCallback(lambda ignored:
 967             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 968                              "400 Bad Request", "read-only cap",
 969                              self.PUT, base, "new_data"))
 970         return d
 971
 972     # TODO: version of this with a Unicode filename
 973     def test_GET_FILEURL_save(self):
 974         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 975                      return_response=True)
 976         def _got((res, statuscode, headers)):
 977             content_disposition = headers["content-disposition"][0]
 978             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 979             self.failUnlessIsBarDotTxt(res)
 980         d.addCallback(_got)
 981         return d
 982
 983     def test_GET_FILEURL_missing(self):
 984         d = self.GET(self.public_url + "/foo/missing")
 985         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 986         return d
 987
 988     def test_GET_FILEURL_info_mdmf(self):
 989         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 990         def _got(res):
 991             self.failUnlessIn("mutable file (mdmf)", res)
 992             self.failUnlessIn(self._quux_txt_uri, res)
 993             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 994         d.addCallback(_got)
 995         return d
 996
 997     def test_GET_FILEURL_info_mdmf_readonly(self):
 998         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 999         def _got(res):
1000             self.failUnlessIn("mutable file (mdmf)", res)
1001             self.failIfIn(self._quux_txt_uri, res)
1002             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1003         d.addCallback(_got)
1004         return d
1005
1006     def test_GET_FILEURL_info_sdmf(self):
1007         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1008         def _got(res):
1009             self.failUnlessIn("mutable file (sdmf)", res)
1010             self.failUnlessIn(self._baz_txt_uri, res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_info_mdmf_extensions(self):
1015         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1016         def _got(res):
1017             self.failUnlessIn("mutable file (mdmf)", res)
1018             self.failUnlessIn(self._quux_txt_uri, res)
1019             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_PUT_overwrite_only_files(self):
1024         # create a directory, put a file in that directory.
1025         contents, n, filecap = self.makefile(8)
1026         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1027         d.addCallback(lambda res:
1028             self.PUT(self.public_url + "/foo/dir/file1.txt",
1029                      self.NEWFILE_CONTENTS))
1030         # try to overwrite the file with replace=only-files
1031         # (this should work)
1032         d.addCallback(lambda res:
1033             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1034                      filecap))
1035         d.addCallback(lambda res:
1036             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1037                  "There was already a child by that name, and you asked me "
1038                  "to not replace it",
1039                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1040                  filecap))
1041         return d
1042
1043     def test_PUT_NEWFILEURL(self):
1044         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1045         # TODO: we lose the response code, so we can't check this
1046         #self.failUnlessReallyEqual(responsecode, 201)
1047         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1048         d.addCallback(lambda res:
1049                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1050                                                       self.NEWFILE_CONTENTS))
1051         return d
1052
1053     def test_PUT_NEWFILEURL_not_mutable(self):
1054         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1055                      self.NEWFILE_CONTENTS)
1056         # TODO: we lose the response code, so we can't check this
1057         #self.failUnlessReallyEqual(responsecode, 201)
1058         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1059         d.addCallback(lambda res:
1060                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1061                                                       self.NEWFILE_CONTENTS))
1062         return d
1063
1064     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1065         # this should get us a few segments of an MDMF mutable file,
1066         # which we can then test for.
1067         contents = self.NEWFILE_CONTENTS * 300000
1068         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1069                      contents)
1070         def _got_filecap(filecap):
1071             self.failUnless(filecap.startswith("URI:MDMF"))
1072             return filecap
1073         d.addCallback(_got_filecap)
1074         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1075         d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1076         return d
1077
1078     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1079         contents = self.NEWFILE_CONTENTS * 300000
1080         d = self.PUT("/uri?mutable=true&mutable-type=sdmf",
1081                      contents)
1082         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1083         d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1084         return d
1085
1086     def test_PUT_NEWFILEURL_unlinked_bad_mutable_type(self):
1087         contents = self.NEWFILE_CONTENTS * 300000
1088         return self.shouldHTTPError("test bad mutable type",
1089                                     400, "Bad Request", "Unknown type: foo",
1090                                     self.PUT, "/uri?mutable=true&mutable-type=foo",
1091                                     contents)
1092
1093     def test_PUT_NEWFILEURL_range_bad(self):
1094         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1095         target = self.public_url + "/foo/new.txt"
1096         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1097                              "501 Not Implemented",
1098                              "Content-Range in PUT not yet supported",
1099                              # (and certainly not for immutable files)
1100                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1101                              headers=headers)
1102         d.addCallback(lambda res:
1103                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1104         return d
1105
1106     def test_PUT_NEWFILEURL_mutable(self):
1107         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1108                      self.NEWFILE_CONTENTS)
1109         # TODO: we lose the response code, so we can't check this
1110         #self.failUnlessReallyEqual(responsecode, 201)
1111         def _check_uri(res):
1112             u = uri.from_string_mutable_filenode(res)
1113             self.failUnless(u.is_mutable())
1114             self.failIf(u.is_readonly())
1115             return res
1116         d.addCallback(_check_uri)
1117         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1118         d.addCallback(lambda res:
1119                       self.failUnlessMutableChildContentsAre(self._foo_node,
1120                                                              u"new.txt",
1121                                                              self.NEWFILE_CONTENTS))
1122         return d
1123
1124     def test_PUT_NEWFILEURL_mutable_toobig(self):
1125         # It is okay to upload large mutable files, so we should be able
1126         # to do that.
1127         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1128                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1129         return d
1130
1131     def test_PUT_NEWFILEURL_replace(self):
1132         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1133         # TODO: we lose the response code, so we can't check this
1134         #self.failUnlessReallyEqual(responsecode, 200)
1135         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1136         d.addCallback(lambda res:
1137                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1138                                                       self.NEWFILE_CONTENTS))
1139         return d
1140
1141     def test_PUT_NEWFILEURL_bad_t(self):
1142         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1143                              "PUT to a file: bad t=bogus",
1144                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1145                              "contents")
1146         return d
1147
1148     def test_PUT_NEWFILEURL_no_replace(self):
1149         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1150                      self.NEWFILE_CONTENTS)
1151         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1152                   "409 Conflict",
1153                   "There was already a child by that name, and you asked me "
1154                   "to not replace it")
1155         return d
1156
1157     def test_PUT_NEWFILEURL_mkdirs(self):
1158         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1159         fn = self._foo_node
1160         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1161         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1162         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1163         d.addCallback(lambda res:
1164                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1165                                                       self.NEWFILE_CONTENTS))
1166         return d
1167
1168     def test_PUT_NEWFILEURL_blocked(self):
1169         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1170                      self.NEWFILE_CONTENTS)
1171         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1172                   "409 Conflict",
1173                   "Unable to create directory 'blockingfile': a file was in the way")
1174         return d
1175
1176     def test_PUT_NEWFILEURL_emptyname(self):
1177         # an empty pathname component (i.e. a double-slash) is disallowed
1178         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1179                              "400 Bad Request",
1180                              "The webapi does not allow empty pathname components",
1181                              self.PUT, self.public_url + "/foo//new.txt", "")
1182         return d
1183
1184     def test_DELETE_FILEURL(self):
1185         d = self.DELETE(self.public_url + "/foo/bar.txt")
1186         d.addCallback(lambda res:
1187                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1188         return d
1189
1190     def test_DELETE_FILEURL_missing(self):
1191         d = self.DELETE(self.public_url + "/foo/missing")
1192         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1193         return d
1194
1195     def test_DELETE_FILEURL_missing2(self):
1196         d = self.DELETE(self.public_url + "/missing/missing")
1197         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1198         return d
1199
1200     def failUnlessHasBarDotTxtMetadata(self, res):
1201         data = simplejson.loads(res)
1202         self.failUnless(isinstance(data, list))
1203         self.failUnlessIn("metadata", data[1])
1204         self.failUnlessIn("tahoe", data[1]["metadata"])
1205         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1206         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1207         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1208                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1209
1210     def test_GET_FILEURL_json(self):
1211         # twisted.web.http.parse_qs ignores any query args without an '=', so
1212         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1213         # instead. This may make it tricky to emulate the S3 interface
1214         # completely.
1215         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1216         def _check1(data):
1217             self.failUnlessIsBarJSON(data)
1218             self.failUnlessHasBarDotTxtMetadata(data)
1219             return
1220         d.addCallback(_check1)
1221         return d
1222
1223     def test_GET_FILEURL_json_mutable_type(self):
1224         # The JSON should include mutable-type, which says whether the
1225         # file is SDMF or MDMF
1226         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1227                      self.NEWFILE_CONTENTS * 300000)
1228         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1229         def _got_json(json, version):
1230             data = simplejson.loads(json)
1231             assert "filenode" == data[0]
1232             data = data[1]
1233             assert isinstance(data, dict)
1234
1235             self.failUnlessIn("mutable-type", data)
1236             self.failUnlessEqual(data['mutable-type'], version)
1237
1238         d.addCallback(_got_json, "mdmf")
1239         # Now make an SDMF file and check that it is reported correctly.
1240         d.addCallback(lambda ignored:
1241             self.PUT("/uri?mutable=true&mutable-type=sdmf",
1242                       self.NEWFILE_CONTENTS * 300000))
1243         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244         d.addCallback(_got_json, "sdmf")
1245         return d
1246
1247     def test_GET_FILEURL_json_mdmf(self):
1248         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1249         d.addCallback(self.failUnlessIsQuuxJSON)
1250         return d
1251
1252     def test_GET_FILEURL_json_missing(self):
1253         d = self.GET(self.public_url + "/foo/missing?json")
1254         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1255         return d
1256
1257     def test_GET_FILEURL_uri(self):
1258         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1259         def _check(res):
1260             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1261         d.addCallback(_check)
1262         d.addCallback(lambda res:
1263                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1264         def _check2(res):
1265             # for now, for files, uris and readonly-uris are the same
1266             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1267         d.addCallback(_check2)
1268         return d
1269
1270     def test_GET_FILEURL_badtype(self):
1271         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1272                                  "bad t=bogus",
1273                                  self.GET,
1274                                  self.public_url + "/foo/bar.txt?t=bogus")
1275         return d
1276
1277     def test_CSS_FILE(self):
1278         d = self.GET("/tahoe_css", followRedirect=True)
1279         def _check(res):
1280             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1281             self.failUnless(CSS_STYLE.search(res), res)
1282         d.addCallback(_check)
1283         return d
1284
1285     def test_GET_FILEURL_uri_missing(self):
1286         d = self.GET(self.public_url + "/foo/missing?t=uri")
1287         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1288         return d
1289
1290     def test_GET_DIRECTORY_html(self):
1291         d = self.GET(self.public_url + "/foo", followRedirect=True)
1292         def _check(res):
1293             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1294             # These are radio buttons that allow a user to toggle
1295             # whether a particular mutable file is SDMF or MDMF.
1296             self.failUnlessIn("mutable-type-mdmf", res)
1297             self.failUnlessIn("mutable-type-sdmf", res)
1298             # Similarly, these toggle whether a particular directory
1299             # should be MDMF or SDMF.
1300             self.failUnlessIn("mutable-directory-mdmf", res)
1301             self.failUnlessIn("mutable-directory-sdmf", res)
1302             self.failUnlessIn("quux", res)
1303         d.addCallback(_check)
1304         return d
1305
1306     def test_GET_root_html(self):
1307         # make sure that we have the option to upload an unlinked
1308         # mutable file in SDMF and MDMF formats.
1309         d = self.GET("/")
1310         def _got_html(html):
1311             # These are radio buttons that allow the user to toggle
1312             # whether a particular mutable file is MDMF or SDMF.
1313             self.failUnlessIn("mutable-type-mdmf", html)
1314             self.failUnlessIn("mutable-type-sdmf", html)
1315             # We should also have the ability to create a mutable directory.
1316             self.failUnlessIn("mkdir", html)
1317             # ...and we should have the ability to say whether that's an
1318             # MDMF or SDMF directory
1319             self.failUnlessIn("mutable-directory-mdmf", html)
1320             self.failUnlessIn("mutable-directory-sdmf", html)
1321         d.addCallback(_got_html)
1322         return d
1323
1324     def test_mutable_type_defaults(self):
1325         # The checked="checked" attribute of the inputs corresponding to
1326         # the mutable-type parameter should change as expected with the
1327         # value configured in tahoe.cfg.
1328         #
1329         # By default, the value configured with the client is
1330         # SDMF_VERSION, so that should be checked.
1331         assert self.s.mutable_file_default == SDMF_VERSION
1332
1333         d = self.GET("/")
1334         def _got_html(html, value):
1335             i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1336             self.failUnlessIn(i % value, html)
1337         d.addCallback(_got_html, "sdmf")
1338         d.addCallback(lambda ignored:
1339             self.GET(self.public_url + "/foo", followRedirect=True))
1340         d.addCallback(_got_html, "sdmf")
1341         # Now switch the configuration value to MDMF. The MDMF radio
1342         # buttons should now be checked on these pages.
1343         def _swap_values(ignored):
1344             self.s.mutable_file_default = MDMF_VERSION
1345         d.addCallback(_swap_values)
1346         d.addCallback(lambda ignored: self.GET("/"))
1347         d.addCallback(_got_html, "mdmf")
1348         d.addCallback(lambda ignored:
1349             self.GET(self.public_url + "/foo", followRedirect=True))
1350         d.addCallback(_got_html, "mdmf")
1351         return d
1352
1353     def test_GET_DIRURL(self):
1354         # the addSlash means we get a redirect here
1355         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1356         ROOT = "../../.."
1357         d = self.GET(self.public_url + "/foo", followRedirect=True)
1358         def _check(res):
1359             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1360                             in res, res)
1361             # the FILE reference points to a URI, but it should end in bar.txt
1362             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1363                        (ROOT, urllib.quote(self._bar_txt_uri)))
1364             get_bar = "".join([r'<td>FILE</td>',
1365                                r'\s+<td>',
1366                                r'<a href="%s">bar.txt</a>' % bar_url,
1367                                r'</td>',
1368                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1369                                ])
1370             self.failUnless(re.search(get_bar, res), res)
1371             for label in ['unlink', 'rename']:
1372                 for line in res.split("\n"):
1373                     # find the line that contains the relevant button for bar.txt
1374                     if ("form action" in line and
1375                         ('value="%s"' % (label,)) in line and
1376                         'value="bar.txt"' in line):
1377                         # the form target should use a relative URL
1378                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1379                         self.failUnlessIn('action="%s"' % foo_url, line)
1380                         # and the when_done= should too
1381                         #done_url = urllib.quote(???)
1382                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1383
1384                         # 'unlink' needs to use POST because it directly has a side effect
1385                         if label == 'unlink':
1386                             self.failUnlessIn('method="post"', line)
1387                         break
1388                 else:
1389                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1390
1391             # the DIR reference just points to a URI
1392             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1393             get_sub = ((r'<td>DIR</td>')
1394                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1395             self.failUnless(re.search(get_sub, res), res)
1396         d.addCallback(_check)
1397
1398         # look at a readonly directory
1399         d.addCallback(lambda res:
1400                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1401         def _check2(res):
1402             self.failUnless("(read-only)" in res, res)
1403             self.failIf("Upload a file" in res, res)
1404         d.addCallback(_check2)
1405
1406         # and at a directory that contains a readonly directory
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url, followRedirect=True))
1409         def _check3(res):
1410             self.failUnless(re.search('<td>DIR-RO</td>'
1411                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1412         d.addCallback(_check3)
1413
1414         # and an empty directory
1415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1416         def _check4(res):
1417             self.failUnless("directory is empty" in res, res)
1418             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1419             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1420         d.addCallback(_check4)
1421
1422         # and at a literal directory
1423         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1424         d.addCallback(lambda res:
1425                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1426         def _check5(res):
1427             self.failUnless('(immutable)' in res, res)
1428             self.failUnless(re.search('<td>FILE</td>'
1429                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1430         d.addCallback(_check5)
1431         return d
1432
1433     def test_GET_DIRURL_badtype(self):
1434         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1435                                  400, "Bad Request",
1436                                  "bad t=bogus",
1437                                  self.GET,
1438                                  self.public_url + "/foo?t=bogus")
1439         return d
1440
1441     def test_GET_DIRURL_json(self):
1442         d = self.GET(self.public_url + "/foo?t=json")
1443         d.addCallback(self.failUnlessIsFooJSON)
1444         return d
1445
1446     def test_GET_DIRURL_json_mutable_type(self):
1447         d = self.PUT(self.public_url + \
1448                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
1449                      self.NEWFILE_CONTENTS * 300000)
1450         d.addCallback(lambda ignored:
1451             self.PUT(self.public_url + \
1452                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
1453                      self.NEWFILE_CONTENTS * 300000))
1454         # Now we have an MDMF and SDMF file in the directory. If we GET
1455         # its JSON, we should see their encodings.
1456         d.addCallback(lambda ignored:
1457             self.GET(self.public_url + "/foo?t=json"))
1458         def _got_json(json):
1459             data = simplejson.loads(json)
1460             assert data[0] == "dirnode"
1461
1462             data = data[1]
1463             kids = data['children']
1464
1465             mdmf_data = kids['mdmf.txt'][1]
1466             self.failUnlessIn("mutable-type", mdmf_data)
1467             self.failUnlessEqual(mdmf_data['mutable-type'], "mdmf")
1468
1469             sdmf_data = kids['sdmf.txt'][1]
1470             self.failUnlessIn("mutable-type", sdmf_data)
1471             self.failUnlessEqual(sdmf_data['mutable-type'], "sdmf")
1472         d.addCallback(_got_json)
1473         return d
1474
1475
1476     def test_POST_DIRURL_manifest_no_ophandle(self):
1477         d = self.shouldFail2(error.Error,
1478                              "test_POST_DIRURL_manifest_no_ophandle",
1479                              "400 Bad Request",
1480                              "slow operation requires ophandle=",
1481                              self.POST, self.public_url, t="start-manifest")
1482         return d
1483
1484     def test_POST_DIRURL_manifest(self):
1485         d = defer.succeed(None)
1486         def getman(ignored, output):
1487             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1488                           followRedirect=True)
1489             d.addCallback(self.wait_for_operation, "125")
1490             d.addCallback(self.get_operation_results, "125", output)
1491             return d
1492         d.addCallback(getman, None)
1493         def _got_html(manifest):
1494             self.failUnless("Manifest of SI=" in manifest)
1495             self.failUnless("<td>sub</td>" in manifest)
1496             self.failUnless(self._sub_uri in manifest)
1497             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1498         d.addCallback(_got_html)
1499
1500         # both t=status and unadorned GET should be identical
1501         d.addCallback(lambda res: self.GET("/operations/125"))
1502         d.addCallback(_got_html)
1503
1504         d.addCallback(getman, "html")
1505         d.addCallback(_got_html)
1506         d.addCallback(getman, "text")
1507         def _got_text(manifest):
1508             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1509             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1510         d.addCallback(_got_text)
1511         d.addCallback(getman, "JSON")
1512         def _got_json(res):
1513             data = res["manifest"]
1514             got = {}
1515             for (path_list, cap) in data:
1516                 got[tuple(path_list)] = cap
1517             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1518             self.failUnless((u"sub",u"baz.txt") in got)
1519             self.failUnless("finished" in res)
1520             self.failUnless("origin" in res)
1521             self.failUnless("storage-index" in res)
1522             self.failUnless("verifycaps" in res)
1523             self.failUnless("stats" in res)
1524         d.addCallback(_got_json)
1525         return d
1526
1527     def test_POST_DIRURL_deepsize_no_ophandle(self):
1528         d = self.shouldFail2(error.Error,
1529                              "test_POST_DIRURL_deepsize_no_ophandle",
1530                              "400 Bad Request",
1531                              "slow operation requires ophandle=",
1532                              self.POST, self.public_url, t="start-deep-size")
1533         return d
1534
1535     def test_POST_DIRURL_deepsize(self):
1536         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1537                       followRedirect=True)
1538         d.addCallback(self.wait_for_operation, "126")
1539         d.addCallback(self.get_operation_results, "126", "json")
1540         def _got_json(data):
1541             self.failUnlessReallyEqual(data["finished"], True)
1542             size = data["size"]
1543             self.failUnless(size > 1000)
1544         d.addCallback(_got_json)
1545         d.addCallback(self.get_operation_results, "126", "text")
1546         def _got_text(res):
1547             mo = re.search(r'^size: (\d+)$', res, re.M)
1548             self.failUnless(mo, res)
1549             size = int(mo.group(1))
1550             # with directories, the size varies.
1551             self.failUnless(size > 1000)
1552         d.addCallback(_got_text)
1553         return d
1554
1555     def test_POST_DIRURL_deepstats_no_ophandle(self):
1556         d = self.shouldFail2(error.Error,
1557                              "test_POST_DIRURL_deepstats_no_ophandle",
1558                              "400 Bad Request",
1559                              "slow operation requires ophandle=",
1560                              self.POST, self.public_url, t="start-deep-stats")
1561         return d
1562
1563     def test_POST_DIRURL_deepstats(self):
1564         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1565                       followRedirect=True)
1566         d.addCallback(self.wait_for_operation, "127")
1567         d.addCallback(self.get_operation_results, "127", "json")
1568         def _got_json(stats):
1569             expected = {"count-immutable-files": 3,
1570                         "count-mutable-files": 2,
1571                         "count-literal-files": 0,
1572                         "count-files": 5,
1573                         "count-directories": 3,
1574                         "size-immutable-files": 57,
1575                         "size-literal-files": 0,
1576                         #"size-directories": 1912, # varies
1577                         #"largest-directory": 1590,
1578                         "largest-directory-children": 7,
1579                         "largest-immutable-file": 19,
1580                         }
1581             for k,v in expected.iteritems():
1582                 self.failUnlessReallyEqual(stats[k], v,
1583                                            "stats[%s] was %s, not %s" %
1584                                            (k, stats[k], v))
1585             self.failUnlessReallyEqual(stats["size-files-histogram"],
1586                                        [ [11, 31, 3] ])
1587         d.addCallback(_got_json)
1588         return d
1589
1590     def test_POST_DIRURL_stream_manifest(self):
1591         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1592         def _check(res):
1593             self.failUnless(res.endswith("\n"))
1594             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1595             self.failUnlessReallyEqual(len(units), 9)
1596             self.failUnlessEqual(units[-1]["type"], "stats")
1597             first = units[0]
1598             self.failUnlessEqual(first["path"], [])
1599             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1600             self.failUnlessEqual(first["type"], "directory")
1601             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1602             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1603             self.failIfEqual(baz["storage-index"], None)
1604             self.failIfEqual(baz["verifycap"], None)
1605             self.failIfEqual(baz["repaircap"], None)
1606             # XXX: Add quux and baz to this test.
1607             return
1608         d.addCallback(_check)
1609         return d
1610
1611     def test_GET_DIRURL_uri(self):
1612         d = self.GET(self.public_url + "/foo?t=uri")
1613         def _check(res):
1614             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1615         d.addCallback(_check)
1616         return d
1617
1618     def test_GET_DIRURL_readonly_uri(self):
1619         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1620         def _check(res):
1621             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1622         d.addCallback(_check)
1623         return d
1624
1625     def test_PUT_NEWDIRURL(self):
1626         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, [])
1631         return d
1632
1633     def test_PUT_NEWDIRURL_mdmf(self):
1634         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1635         d.addCallback(lambda res:
1636                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(lambda node:
1639             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1640         return d
1641
1642     def test_PUT_NEWDIRURL_sdmf(self):
1643         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf",
1644                      "")
1645         d.addCallback(lambda res:
1646                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648         d.addCallback(lambda node:
1649             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1650         return d
1651
1652     def test_PUT_NEWDIRURL_bad_mutable_type(self):
1653         return self.shouldHTTPError("test bad mutable type",
1654                              400, "Bad Request", "Unknown type: foo",
1655                              self.PUT, self.public_url + \
1656                              "/foo/newdir=?t=mkdir&mutable-type=foo", "")
1657
1658     def test_POST_NEWDIRURL(self):
1659         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1660         d.addCallback(lambda res:
1661                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1662         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1663         d.addCallback(self.failUnlessNodeKeysAre, [])
1664         return d
1665
1666     def test_POST_NEWDIRURL_mdmf(self):
1667         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1668         d.addCallback(lambda res:
1669                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1670         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1671         d.addCallback(lambda node:
1672             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1673         return d
1674
1675     def test_POST_NEWDIRURL_sdmf(self):
1676         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf", "")
1677         d.addCallback(lambda res:
1678             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1679         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680         d.addCallback(lambda node:
1681             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1682         return d
1683
1684     def test_POST_NEWDIRURL_bad_mutable_type(self):
1685         return self.shouldHTTPError("test bad mutable type",
1686                                     400, "Bad Request", "Unknown type: foo",
1687                                     self.POST2, self.public_url + \
1688                                     "/foo/newdir?t=mkdir&mutable-type=foo", "")
1689
1690     def test_POST_NEWDIRURL_emptyname(self):
1691         # an empty pathname component (i.e. a double-slash) is disallowed
1692         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1693                              "400 Bad Request",
1694                              "The webapi does not allow empty pathname components, i.e. a double slash",
1695                              self.POST, self.public_url + "//?t=mkdir")
1696         return d
1697
1698     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1699         (newkids, caps) = self._create_initial_children()
1700         query = "/foo/newdir?t=mkdir-with-children"
1701         if version == MDMF_VERSION:
1702             query += "&mutable-type=mdmf"
1703         elif version == SDMF_VERSION:
1704             query += "&mutable-type=sdmf"
1705         else:
1706             version = SDMF_VERSION # for later
1707         d = self.POST2(self.public_url + query,
1708                        simplejson.dumps(newkids))
1709         def _check(uri):
1710             n = self.s.create_node_from_uri(uri.strip())
1711             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1712             self.failUnlessEqual(n._node.get_version(), version)
1713             d2.addCallback(lambda ign:
1714                            self.failUnlessROChildURIIs(n, u"child-imm",
1715                                                        caps['filecap1']))
1716             d2.addCallback(lambda ign:
1717                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1718                                                        caps['filecap2']))
1719             d2.addCallback(lambda ign:
1720                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1721                                                        caps['filecap3']))
1722             d2.addCallback(lambda ign:
1723                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1724                                                        caps['unknown_rocap']))
1725             d2.addCallback(lambda ign:
1726                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1727                                                        caps['unknown_rwcap']))
1728             d2.addCallback(lambda ign:
1729                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1730                                                        caps['unknown_immcap']))
1731             d2.addCallback(lambda ign:
1732                            self.failUnlessRWChildURIIs(n, u"dirchild",
1733                                                        caps['dircap']))
1734             d2.addCallback(lambda ign:
1735                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1736                                                        caps['litdircap']))
1737             d2.addCallback(lambda ign:
1738                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1739                                                        caps['emptydircap']))
1740             return d2
1741         d.addCallback(_check)
1742         d.addCallback(lambda res:
1743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1748         return d
1749
1750     def test_POST_NEWDIRURL_initial_children(self):
1751         return self._do_POST_NEWDIRURL_initial_children_test()
1752
1753     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1754         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1755
1756     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1757         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1758
1759     def test_POST_NEWDIRURL_initial_children_bad_mutable_type(self):
1760         (newkids, caps) = self._create_initial_children()
1761         return self.shouldHTTPError("test bad mutable type",
1762                                     400, "Bad Request", "Unknown type: foo",
1763                                     self.POST2, self.public_url + \
1764                                     "/foo/newdir?t=mkdir-with-children&mutable-type=foo",
1765                                     simplejson.dumps(newkids))
1766
1767     def test_POST_NEWDIRURL_immutable(self):
1768         (newkids, caps) = self._create_immutable_children()
1769         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1770                        simplejson.dumps(newkids))
1771         def _check(uri):
1772             n = self.s.create_node_from_uri(uri.strip())
1773             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1774             d2.addCallback(lambda ign:
1775                            self.failUnlessROChildURIIs(n, u"child-imm",
1776                                                        caps['filecap1']))
1777             d2.addCallback(lambda ign:
1778                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1779                                                        caps['unknown_immcap']))
1780             d2.addCallback(lambda ign:
1781                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1782                                                        caps['immdircap']))
1783             d2.addCallback(lambda ign:
1784                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1785                                                        caps['litdircap']))
1786             d2.addCallback(lambda ign:
1787                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1788                                                        caps['emptydircap']))
1789             return d2
1790         d.addCallback(_check)
1791         d.addCallback(lambda res:
1792                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1795         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1796         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1800         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1802         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1805         d.addErrback(self.explain_web_error)
1806         return d
1807
1808     def test_POST_NEWDIRURL_immutable_bad(self):
1809         (newkids, caps) = self._create_initial_children()
1810         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1811                              "400 Bad Request",
1812                              "needed to be immutable but was not",
1813                              self.POST2,
1814                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1815                              simplejson.dumps(newkids))
1816         return d
1817
1818     def test_PUT_NEWDIRURL_exists(self):
1819         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1822         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1823         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1824         return d
1825
1826     def test_PUT_NEWDIRURL_blocked(self):
1827         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1828                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1829                              self.PUT,
1830                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1833         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1834         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mkdir_p(self):
1838         d = defer.succeed(None)
1839         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1840         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1841         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1842         def mkdir_p(mkpnode):
1843             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1844             d = self.POST(url)
1845             def made_subsub(ssuri):
1846                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1847                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1848                 d = self.POST(url)
1849                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1850                 return d
1851             d.addCallback(made_subsub)
1852             return d
1853         d.addCallback(mkdir_p)
1854         return d
1855
1856     def test_PUT_NEWDIRURL_mkdirs(self):
1857         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1858         d.addCallback(lambda res:
1859                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1860         d.addCallback(lambda res:
1861                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1862         d.addCallback(lambda res:
1863                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1864         d.addCallback(self.failUnlessNodeKeysAre, [])
1865         return d
1866
1867     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1868         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=mdmf", "")
1869         d.addCallback(lambda ignored:
1870             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1871         d.addCallback(lambda ignored:
1872             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1873         d.addCallback(lambda ignored:
1874             self._foo_node.get_child_at_path(u"subdir"))
1875         def _got_subdir(subdir):
1876             # XXX: What we want?
1877             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1878             self.failUnlessNodeHasChild(subdir, u"newdir")
1879             return subdir.get_child_at_path(u"newdir")
1880         d.addCallback(_got_subdir)
1881         d.addCallback(lambda newdir:
1882             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1883         return d
1884
1885     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1886         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=sdmf", "")
1887         d.addCallback(lambda ignored:
1888             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1889         d.addCallback(lambda ignored:
1890             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1891         d.addCallback(lambda ignored:
1892             self._foo_node.get_child_at_path(u"subdir"))
1893         def _got_subdir(subdir):
1894             # XXX: What we want?
1895             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1896             self.failUnlessNodeHasChild(subdir, u"newdir")
1897             return subdir.get_child_at_path(u"newdir")
1898         d.addCallback(_got_subdir)
1899         d.addCallback(lambda newdir:
1900             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1901         return d
1902
1903     def test_PUT_NEWDIRURL_mkdirs_bad_mutable_type(self):
1904         return self.shouldHTTPError("test bad mutable type",
1905                                     400, "Bad Request", "Unknown type: foo",
1906                                     self.PUT, self.public_url + \
1907                                     "/foo/subdir/newdir?t=mkdir&mutable-type=foo",
1908                                     "")
1909
1910     def test_DELETE_DIRURL(self):
1911         d = self.DELETE(self.public_url + "/foo")
1912         d.addCallback(lambda res:
1913                       self.failIfNodeHasChild(self.public_root, u"foo"))
1914         return d
1915
1916     def test_DELETE_DIRURL_missing(self):
1917         d = self.DELETE(self.public_url + "/foo/missing")
1918         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1919         d.addCallback(lambda res:
1920                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1921         return d
1922
1923     def test_DELETE_DIRURL_missing2(self):
1924         d = self.DELETE(self.public_url + "/missing")
1925         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1926         return d
1927
1928     def dump_root(self):
1929         print "NODEWALK"
1930         w = webish.DirnodeWalkerMixin()
1931         def visitor(childpath, childnode, metadata):
1932             print childpath
1933         d = w.walk(self.public_root, visitor)
1934         return d
1935
1936     def failUnlessNodeKeysAre(self, node, expected_keys):
1937         for k in expected_keys:
1938             assert isinstance(k, unicode)
1939         d = node.list()
1940         def _check(children):
1941             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1942         d.addCallback(_check)
1943         return d
1944     def failUnlessNodeHasChild(self, node, name):
1945         assert isinstance(name, unicode)
1946         d = node.list()
1947         def _check(children):
1948             self.failUnless(name in children)
1949         d.addCallback(_check)
1950         return d
1951     def failIfNodeHasChild(self, node, name):
1952         assert isinstance(name, unicode)
1953         d = node.list()
1954         def _check(children):
1955             self.failIf(name in children)
1956         d.addCallback(_check)
1957         return d
1958
1959     def failUnlessChildContentsAre(self, node, name, expected_contents):
1960         assert isinstance(name, unicode)
1961         d = node.get_child_at_path(name)
1962         d.addCallback(lambda node: download_to_data(node))
1963         def _check(contents):
1964             self.failUnlessReallyEqual(contents, expected_contents)
1965         d.addCallback(_check)
1966         return d
1967
1968     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1969         assert isinstance(name, unicode)
1970         d = node.get_child_at_path(name)
1971         d.addCallback(lambda node: node.download_best_version())
1972         def _check(contents):
1973             self.failUnlessReallyEqual(contents, expected_contents)
1974         d.addCallback(_check)
1975         return d
1976
1977     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1978         assert isinstance(name, unicode)
1979         d = node.get_child_at_path(name)
1980         def _check(child):
1981             self.failUnless(child.is_unknown() or not child.is_readonly())
1982             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1983             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1984             expected_ro_uri = self._make_readonly(expected_uri)
1985             if expected_ro_uri:
1986                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1987         d.addCallback(_check)
1988         return d
1989
1990     def failUnlessROChildURIIs(self, node, name, expected_uri):
1991         assert isinstance(name, unicode)
1992         d = node.get_child_at_path(name)
1993         def _check(child):
1994             self.failUnless(child.is_unknown() or child.is_readonly())
1995             self.failUnlessReallyEqual(child.get_write_uri(), None)
1996             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1997             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1998         d.addCallback(_check)
1999         return d
2000
2001     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2002         assert isinstance(name, unicode)
2003         d = node.get_child_at_path(name)
2004         def _check(child):
2005             self.failUnless(child.is_unknown() or not child.is_readonly())
2006             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2007             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2008             expected_ro_uri = self._make_readonly(got_uri)
2009             if expected_ro_uri:
2010                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2011         d.addCallback(_check)
2012         return d
2013
2014     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2015         assert isinstance(name, unicode)
2016         d = node.get_child_at_path(name)
2017         def _check(child):
2018             self.failUnless(child.is_unknown() or child.is_readonly())
2019             self.failUnlessReallyEqual(child.get_write_uri(), None)
2020             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2021             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2022         d.addCallback(_check)
2023         return d
2024
2025     def failUnlessCHKURIHasContents(self, got_uri, contents):
2026         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2027
2028     def test_POST_upload(self):
2029         d = self.POST(self.public_url + "/foo", t="upload",
2030                       file=("new.txt", self.NEWFILE_CONTENTS))
2031         fn = self._foo_node
2032         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2033         d.addCallback(lambda res:
2034                       self.failUnlessChildContentsAre(fn, u"new.txt",
2035                                                       self.NEWFILE_CONTENTS))
2036         return d
2037
2038     def test_POST_upload_unicode(self):
2039         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2040         d = self.POST(self.public_url + "/foo", t="upload",
2041                       file=(filename, self.NEWFILE_CONTENTS))
2042         fn = self._foo_node
2043         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2044         d.addCallback(lambda res:
2045                       self.failUnlessChildContentsAre(fn, filename,
2046                                                       self.NEWFILE_CONTENTS))
2047         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2048         d.addCallback(lambda res: self.GET(target_url))
2049         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2050                                                                   self.NEWFILE_CONTENTS,
2051                                                                   contents))
2052         return d
2053
2054     def test_POST_upload_unicode_named(self):
2055         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2056         d = self.POST(self.public_url + "/foo", t="upload",
2057                       name=filename,
2058                       file=("overridden", self.NEWFILE_CONTENTS))
2059         fn = self._foo_node
2060         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2061         d.addCallback(lambda res:
2062                       self.failUnlessChildContentsAre(fn, filename,
2063                                                       self.NEWFILE_CONTENTS))
2064         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2065         d.addCallback(lambda res: self.GET(target_url))
2066         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2067                                                                   self.NEWFILE_CONTENTS,
2068                                                                   contents))
2069         return d
2070
2071     def test_POST_upload_no_link(self):
2072         d = self.POST("/uri", t="upload",
2073                       file=("new.txt", self.NEWFILE_CONTENTS))
2074         def _check_upload_results(page):
2075             # this should be a page which describes the results of the upload
2076             # that just finished.
2077             self.failUnless("Upload Results:" in page)
2078             self.failUnless("URI:" in page)
2079             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2080             mo = uri_re.search(page)
2081             self.failUnless(mo, page)
2082             new_uri = mo.group(1)
2083             return new_uri
2084         d.addCallback(_check_upload_results)
2085         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2086         return d
2087
2088     def test_POST_upload_no_link_whendone(self):
2089         d = self.POST("/uri", t="upload", when_done="/",
2090                       file=("new.txt", self.NEWFILE_CONTENTS))
2091         d.addBoth(self.shouldRedirect, "/")
2092         return d
2093
2094     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2095         d = defer.maybeDeferred(callable, *args, **kwargs)
2096         def done(res):
2097             if isinstance(res, failure.Failure):
2098                 res.trap(error.PageRedirect)
2099                 statuscode = res.value.status
2100                 target = res.value.location
2101                 return checker(statuscode, target)
2102             self.fail("%s: callable was supposed to redirect, not return '%s'"
2103                       % (which, res))
2104         d.addBoth(done)
2105         return d
2106
2107     def test_POST_upload_no_link_whendone_results(self):
2108         def check(statuscode, target):
2109             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2110             self.failUnless(target.startswith(self.webish_url), target)
2111             return client.getPage(target, method="GET")
2112         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2113                                  check,
2114                                  self.POST, "/uri", t="upload",
2115                                  when_done="/uri/%(uri)s",
2116                                  file=("new.txt", self.NEWFILE_CONTENTS))
2117         d.addCallback(lambda res:
2118                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2119         return d
2120
2121     def test_POST_upload_no_link_mutable(self):
2122         d = self.POST("/uri", t="upload", mutable="true",
2123                       file=("new.txt", self.NEWFILE_CONTENTS))
2124         def _check(filecap):
2125             filecap = filecap.strip()
2126             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2127             self.filecap = filecap
2128             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2129             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2130             n = self.s.create_node_from_uri(filecap)
2131             return n.download_best_version()
2132         d.addCallback(_check)
2133         def _check2(data):
2134             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2135             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2136         d.addCallback(_check2)
2137         def _check3(data):
2138             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2139             return self.GET("/file/%s" % urllib.quote(self.filecap))
2140         d.addCallback(_check3)
2141         def _check4(data):
2142             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2143         d.addCallback(_check4)
2144         return d
2145
2146     def test_POST_upload_no_link_mutable_toobig(self):
2147         # The SDMF size limit is no longer in place, so we should be
2148         # able to upload mutable files that are as large as we want them
2149         # to be.
2150         d = self.POST("/uri", t="upload", mutable="true",
2151                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2152         return d
2153
2154
2155     def test_POST_upload_mutable_type_unlinked(self):
2156         d = self.POST("/uri?t=upload&mutable=true&mutable-type=sdmf",
2157                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2158         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2159         def _got_json(json, version):
2160             data = simplejson.loads(json)
2161             data = data[1]
2162
2163             self.failUnlessIn("mutable-type", data)
2164             self.failUnlessEqual(data['mutable-type'], version)
2165         d.addCallback(_got_json, "sdmf")
2166         d.addCallback(lambda ignored:
2167             self.POST("/uri?t=upload&mutable=true&mutable-type=mdmf",
2168                       file=('mdmf.txt', self.NEWFILE_CONTENTS * 300000)))
2169         def _got_filecap(filecap):
2170             self.failUnless(filecap.startswith("URI:MDMF"))
2171             return filecap
2172         d.addCallback(_got_filecap)
2173         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2174         d.addCallback(_got_json, "mdmf")
2175         return d
2176
2177     def test_POST_upload_mutable_type_unlinked_bad_mutable_type(self):
2178         return self.shouldHTTPError("test bad mutable type",
2179                                     400, "Bad Request", "Unknown type: foo",
2180                                     self.POST,
2181                                     "/uri?5=upload&mutable=true&mutable-type=foo",
2182                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2183
2184     def test_POST_upload_mutable_type(self):
2185         d = self.POST(self.public_url + \
2186                       "/foo?t=upload&mutable=true&mutable-type=sdmf",
2187                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2188         fn = self._foo_node
2189         def _got_cap(filecap, filename):
2190             filenameu = unicode(filename)
2191             self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2192             return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2193         def _got_mdmf_cap(filecap):
2194             self.failUnless(filecap.startswith("URI:MDMF"))
2195             return filecap
2196         d.addCallback(_got_cap, "sdmf.txt")
2197         def _got_json(json, version):
2198             data = simplejson.loads(json)
2199             data = data[1]
2200
2201             self.failUnlessIn("mutable-type", data)
2202             self.failUnlessEqual(data['mutable-type'], version)
2203         d.addCallback(_got_json, "sdmf")
2204         d.addCallback(lambda ignored:
2205             self.POST(self.public_url + \
2206                       "/foo?t=upload&mutable=true&mutable-type=mdmf",
2207                       file=("mdmf.txt", self.NEWFILE_CONTENTS * 300000)))
2208         d.addCallback(_got_mdmf_cap)
2209         d.addCallback(_got_cap, "mdmf.txt")
2210         d.addCallback(_got_json, "mdmf")
2211         return d
2212
2213     def test_POST_upload_bad_mutable_type(self):
2214         return self.shouldHTTPError("test bad mutable type",
2215                                     400, "Bad Request", "Unknown type: foo",
2216                                     self.POST, self.public_url + \
2217                                     "/foo?t=upload&mutable=true&mutable-type=foo",
2218                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2219
2220     def test_POST_upload_mutable(self):
2221         # this creates a mutable file
2222         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2223                       file=("new.txt", self.NEWFILE_CONTENTS))
2224         fn = self._foo_node
2225         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2226         d.addCallback(lambda res:
2227                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2228                                                              self.NEWFILE_CONTENTS))
2229         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2230         def _got(newnode):
2231             self.failUnless(IMutableFileNode.providedBy(newnode))
2232             self.failUnless(newnode.is_mutable())
2233             self.failIf(newnode.is_readonly())
2234             self._mutable_node = newnode
2235             self._mutable_uri = newnode.get_uri()
2236         d.addCallback(_got)
2237
2238         # now upload it again and make sure that the URI doesn't change
2239         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2240         d.addCallback(lambda res:
2241                       self.POST(self.public_url + "/foo", t="upload",
2242                                 mutable="true",
2243                                 file=("new.txt", NEWER_CONTENTS)))
2244         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2245         d.addCallback(lambda res:
2246                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2247                                                              NEWER_CONTENTS))
2248         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2249         def _got2(newnode):
2250             self.failUnless(IMutableFileNode.providedBy(newnode))
2251             self.failUnless(newnode.is_mutable())
2252             self.failIf(newnode.is_readonly())
2253             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2254         d.addCallback(_got2)
2255
2256         # upload a second time, using PUT instead of POST
2257         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2258         d.addCallback(lambda res:
2259                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2260         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2261         d.addCallback(lambda res:
2262                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2263                                                              NEW2_CONTENTS))
2264
2265         # finally list the directory, since mutable files are displayed
2266         # slightly differently
2267
2268         d.addCallback(lambda res:
2269                       self.GET(self.public_url + "/foo/",
2270                                followRedirect=True))
2271         def _check_page(res):
2272             # TODO: assert more about the contents
2273             self.failUnless("SSK" in res)
2274             return res
2275         d.addCallback(_check_page)
2276
2277         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2278         def _got3(newnode):
2279             self.failUnless(IMutableFileNode.providedBy(newnode))
2280             self.failUnless(newnode.is_mutable())
2281             self.failIf(newnode.is_readonly())
2282             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2283         d.addCallback(_got3)
2284
2285         # look at the JSON form of the enclosing directory
2286         d.addCallback(lambda res:
2287                       self.GET(self.public_url + "/foo/?t=json",
2288                                followRedirect=True))
2289         def _check_page_json(res):
2290             parsed = simplejson.loads(res)
2291             self.failUnlessEqual(parsed[0], "dirnode")
2292             children = dict( [(unicode(name),value)
2293                               for (name,value)
2294                               in parsed[1]["children"].iteritems()] )
2295             self.failUnless(u"new.txt" in children)
2296             new_json = children[u"new.txt"]
2297             self.failUnlessEqual(new_json[0], "filenode")
2298             self.failUnless(new_json[1]["mutable"])
2299             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2300             ro_uri = self._mutable_node.get_readonly().to_string()
2301             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2302         d.addCallback(_check_page_json)
2303
2304         # and the JSON form of the file
2305         d.addCallback(lambda res:
2306                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2307         def _check_file_json(res):
2308             parsed = simplejson.loads(res)
2309             self.failUnlessEqual(parsed[0], "filenode")
2310             self.failUnless(parsed[1]["mutable"])
2311             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2312             ro_uri = self._mutable_node.get_readonly().to_string()
2313             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2314         d.addCallback(_check_file_json)
2315
2316         # and look at t=uri and t=readonly-uri
2317         d.addCallback(lambda res:
2318                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2319         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2320         d.addCallback(lambda res:
2321                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2322         def _check_ro_uri(res):
2323             ro_uri = self._mutable_node.get_readonly().to_string()
2324             self.failUnlessReallyEqual(res, ro_uri)
2325         d.addCallback(_check_ro_uri)
2326
2327         # make sure we can get to it from /uri/URI
2328         d.addCallback(lambda res:
2329                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2330         d.addCallback(lambda res:
2331                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2332
2333         # and that HEAD computes the size correctly
2334         d.addCallback(lambda res:
2335                       self.HEAD(self.public_url + "/foo/new.txt",
2336                                 return_response=True))
2337         def _got_headers((res, status, headers)):
2338             self.failUnlessReallyEqual(res, "")
2339             self.failUnlessReallyEqual(headers["content-length"][0],
2340                                        str(len(NEW2_CONTENTS)))
2341             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2342         d.addCallback(_got_headers)
2343
2344         # make sure that outdated size limits aren't enforced anymore.
2345         d.addCallback(lambda ignored:
2346             self.POST(self.public_url + "/foo", t="upload",
2347                       mutable="true",
2348                       file=("new.txt",
2349                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2350         d.addErrback(self.dump_error)
2351         return d
2352
2353     def test_POST_upload_mutable_toobig(self):
2354         # SDMF had a size limti that was removed a while ago. MDMF has
2355         # never had a size limit. Test to make sure that we do not
2356         # encounter errors when trying to upload large mutable files,
2357         # since there should be no coded prohibitions regarding large
2358         # mutable files.
2359         d = self.POST(self.public_url + "/foo",
2360                       t="upload", mutable="true",
2361                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2362         return d
2363
2364     def dump_error(self, f):
2365         # if the web server returns an error code (like 400 Bad Request),
2366         # web.client.getPage puts the HTTP response body into the .response
2367         # attribute of the exception object that it gives back. It does not
2368         # appear in the Failure's repr(), so the ERROR that trial displays
2369         # will be rather terse and unhelpful. addErrback this method to the
2370         # end of your chain to get more information out of these errors.
2371         if f.check(error.Error):
2372             print "web.error.Error:"
2373             print f
2374             print f.value.response
2375         return f
2376
2377     def test_POST_upload_replace(self):
2378         d = self.POST(self.public_url + "/foo", t="upload",
2379                       file=("bar.txt", self.NEWFILE_CONTENTS))
2380         fn = self._foo_node
2381         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2382         d.addCallback(lambda res:
2383                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2384                                                       self.NEWFILE_CONTENTS))
2385         return d
2386
2387     def test_POST_upload_no_replace_ok(self):
2388         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2389                       file=("new.txt", self.NEWFILE_CONTENTS))
2390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2391         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2392                                                              self.NEWFILE_CONTENTS))
2393         return d
2394
2395     def test_POST_upload_no_replace_queryarg(self):
2396         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2397                       file=("bar.txt", self.NEWFILE_CONTENTS))
2398         d.addBoth(self.shouldFail, error.Error,
2399                   "POST_upload_no_replace_queryarg",
2400                   "409 Conflict",
2401                   "There was already a child by that name, and you asked me "
2402                   "to not replace it")
2403         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2404         d.addCallback(self.failUnlessIsBarDotTxt)
2405         return d
2406
2407     def test_POST_upload_no_replace_field(self):
2408         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2409                       file=("bar.txt", self.NEWFILE_CONTENTS))
2410         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2411                   "409 Conflict",
2412                   "There was already a child by that name, and you asked me "
2413                   "to not replace it")
2414         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2415         d.addCallback(self.failUnlessIsBarDotTxt)
2416         return d
2417
2418     def test_POST_upload_whendone(self):
2419         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2420                       file=("new.txt", self.NEWFILE_CONTENTS))
2421         d.addBoth(self.shouldRedirect, "/THERE")
2422         fn = self._foo_node
2423         d.addCallback(lambda res:
2424                       self.failUnlessChildContentsAre(fn, u"new.txt",
2425                                                       self.NEWFILE_CONTENTS))
2426         return d
2427
2428     def test_POST_upload_named(self):
2429         fn = self._foo_node
2430         d = self.POST(self.public_url + "/foo", t="upload",
2431                       name="new.txt", file=self.NEWFILE_CONTENTS)
2432         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2433         d.addCallback(lambda res:
2434                       self.failUnlessChildContentsAre(fn, u"new.txt",
2435                                                       self.NEWFILE_CONTENTS))
2436         return d
2437
2438     def test_POST_upload_named_badfilename(self):
2439         d = self.POST(self.public_url + "/foo", t="upload",
2440                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2441         d.addBoth(self.shouldFail, error.Error,
2442                   "test_POST_upload_named_badfilename",
2443                   "400 Bad Request",
2444                   "name= may not contain a slash",
2445                   )
2446         # make sure that nothing was added
2447         d.addCallback(lambda res:
2448                       self.failUnlessNodeKeysAre(self._foo_node,
2449                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2450                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2451                                                   u"sub"]))
2452         return d
2453
2454     def test_POST_FILEURL_check(self):
2455         bar_url = self.public_url + "/foo/bar.txt"
2456         d = self.POST(bar_url, t="check")
2457         def _check(res):
2458             self.failUnless("Healthy :" in res)
2459         d.addCallback(_check)
2460         redir_url = "http://allmydata.org/TARGET"
2461         def _check2(statuscode, target):
2462             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2463             self.failUnlessReallyEqual(target, redir_url)
2464         d.addCallback(lambda res:
2465                       self.shouldRedirect2("test_POST_FILEURL_check",
2466                                            _check2,
2467                                            self.POST, bar_url,
2468                                            t="check",
2469                                            when_done=redir_url))
2470         d.addCallback(lambda res:
2471                       self.POST(bar_url, t="check", return_to=redir_url))
2472         def _check3(res):
2473             self.failUnless("Healthy :" in res)
2474             self.failUnless("Return to file" in res)
2475             self.failUnless(redir_url in res)
2476         d.addCallback(_check3)
2477
2478         d.addCallback(lambda res:
2479                       self.POST(bar_url, t="check", output="JSON"))
2480         def _check_json(res):
2481             data = simplejson.loads(res)
2482             self.failUnless("storage-index" in data)
2483             self.failUnless(data["results"]["healthy"])
2484         d.addCallback(_check_json)
2485
2486         return d
2487
2488     def test_POST_FILEURL_check_and_repair(self):
2489         bar_url = self.public_url + "/foo/bar.txt"
2490         d = self.POST(bar_url, t="check", repair="true")
2491         def _check(res):
2492             self.failUnless("Healthy :" in res)
2493         d.addCallback(_check)
2494         redir_url = "http://allmydata.org/TARGET"
2495         def _check2(statuscode, target):
2496             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2497             self.failUnlessReallyEqual(target, redir_url)
2498         d.addCallback(lambda res:
2499                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2500                                            _check2,
2501                                            self.POST, bar_url,
2502                                            t="check", repair="true",
2503                                            when_done=redir_url))
2504         d.addCallback(lambda res:
2505                       self.POST(bar_url, t="check", return_to=redir_url))
2506         def _check3(res):
2507             self.failUnless("Healthy :" in res)
2508             self.failUnless("Return to file" in res)
2509             self.failUnless(redir_url in res)
2510         d.addCallback(_check3)
2511         return d
2512
2513     def test_POST_DIRURL_check(self):
2514         foo_url = self.public_url + "/foo/"
2515         d = self.POST(foo_url, t="check")
2516         def _check(res):
2517             self.failUnless("Healthy :" in res, res)
2518         d.addCallback(_check)
2519         redir_url = "http://allmydata.org/TARGET"
2520         def _check2(statuscode, target):
2521             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2522             self.failUnlessReallyEqual(target, redir_url)
2523         d.addCallback(lambda res:
2524                       self.shouldRedirect2("test_POST_DIRURL_check",
2525                                            _check2,
2526                                            self.POST, foo_url,
2527                                            t="check",
2528                                            when_done=redir_url))
2529         d.addCallback(lambda res:
2530                       self.POST(foo_url, t="check", return_to=redir_url))
2531         def _check3(res):
2532             self.failUnless("Healthy :" in res, res)
2533             self.failUnless("Return to file/directory" in res)
2534             self.failUnless(redir_url in res)
2535         d.addCallback(_check3)
2536
2537         d.addCallback(lambda res:
2538                       self.POST(foo_url, t="check", output="JSON"))
2539         def _check_json(res):
2540             data = simplejson.loads(res)
2541             self.failUnless("storage-index" in data)
2542             self.failUnless(data["results"]["healthy"])
2543         d.addCallback(_check_json)
2544
2545         return d
2546
2547     def test_POST_DIRURL_check_and_repair(self):
2548         foo_url = self.public_url + "/foo/"
2549         d = self.POST(foo_url, t="check", repair="true")
2550         def _check(res):
2551             self.failUnless("Healthy :" in res, res)
2552         d.addCallback(_check)
2553         redir_url = "http://allmydata.org/TARGET"
2554         def _check2(statuscode, target):
2555             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2556             self.failUnlessReallyEqual(target, redir_url)
2557         d.addCallback(lambda res:
2558                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2559                                            _check2,
2560                                            self.POST, foo_url,
2561                                            t="check", repair="true",
2562                                            when_done=redir_url))
2563         d.addCallback(lambda res:
2564                       self.POST(foo_url, t="check", return_to=redir_url))
2565         def _check3(res):
2566             self.failUnless("Healthy :" in res)
2567             self.failUnless("Return to file/directory" in res)
2568             self.failUnless(redir_url in res)
2569         d.addCallback(_check3)
2570         return d
2571
2572     def test_POST_FILEURL_mdmf_check(self):
2573         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2574         d = self.POST(quux_url, t="check")
2575         def _check(res):
2576             self.failUnlessIn("Healthy", res)
2577         d.addCallback(_check)
2578         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2579         d.addCallback(lambda ignored:
2580             self.POST(quux_extension_url, t="check"))
2581         d.addCallback(_check)
2582         return d
2583
2584     def test_POST_FILEURL_mdmf_check_and_repair(self):
2585         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2586         d = self.POST(quux_url, t="check", repair="true")
2587         def _check(res):
2588             self.failUnlessIn("Healthy", res)
2589         d.addCallback(_check)
2590         quux_extension_url = "/uri/%s" %\
2591             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2592         d.addCallback(lambda ignored:
2593             self.POST(quux_extension_url, t="check", repair="true"))
2594         d.addCallback(_check)
2595         return d
2596
2597     def wait_for_operation(self, ignored, ophandle):
2598         url = "/operations/" + ophandle
2599         url += "?t=status&output=JSON"
2600         d = self.GET(url)
2601         def _got(res):
2602             data = simplejson.loads(res)
2603             if not data["finished"]:
2604                 d = self.stall(delay=1.0)
2605                 d.addCallback(self.wait_for_operation, ophandle)
2606                 return d
2607             return data
2608         d.addCallback(_got)
2609         return d
2610
2611     def get_operation_results(self, ignored, ophandle, output=None):
2612         url = "/operations/" + ophandle
2613         url += "?t=status"
2614         if output:
2615             url += "&output=" + output
2616         d = self.GET(url)
2617         def _got(res):
2618             if output and output.lower() == "json":
2619                 return simplejson.loads(res)
2620             return res
2621         d.addCallback(_got)
2622         return d
2623
2624     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2625         d = self.shouldFail2(error.Error,
2626                              "test_POST_DIRURL_deepcheck_no_ophandle",
2627                              "400 Bad Request",
2628                              "slow operation requires ophandle=",
2629                              self.POST, self.public_url, t="start-deep-check")
2630         return d
2631
2632     def test_POST_DIRURL_deepcheck(self):
2633         def _check_redirect(statuscode, target):
2634             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2635             self.failUnless(target.endswith("/operations/123"))
2636         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2637                                  self.POST, self.public_url,
2638                                  t="start-deep-check", ophandle="123")
2639         d.addCallback(self.wait_for_operation, "123")
2640         def _check_json(data):
2641             self.failUnlessReallyEqual(data["finished"], True)
2642             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2643             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2644         d.addCallback(_check_json)
2645         d.addCallback(self.get_operation_results, "123", "html")
2646         def _check_html(res):
2647             self.failUnless("Objects Checked: <span>10</span>" in res)
2648             self.failUnless("Objects Healthy: <span>10</span>" in res)
2649         d.addCallback(_check_html)
2650
2651         d.addCallback(lambda res:
2652                       self.GET("/operations/123/"))
2653         d.addCallback(_check_html) # should be the same as without the slash
2654
2655         d.addCallback(lambda res:
2656                       self.shouldFail2(error.Error, "one", "404 Not Found",
2657                                        "No detailed results for SI bogus",
2658                                        self.GET, "/operations/123/bogus"))
2659
2660         foo_si = self._foo_node.get_storage_index()
2661         foo_si_s = base32.b2a(foo_si)
2662         d.addCallback(lambda res:
2663                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2664         def _check_foo_json(res):
2665             data = simplejson.loads(res)
2666             self.failUnlessEqual(data["storage-index"], foo_si_s)
2667             self.failUnless(data["results"]["healthy"])
2668         d.addCallback(_check_foo_json)
2669         return d
2670
2671     def test_POST_DIRURL_deepcheck_and_repair(self):
2672         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2673                       ophandle="124", output="json", followRedirect=True)
2674         d.addCallback(self.wait_for_operation, "124")
2675         def _check_json(data):
2676             self.failUnlessReallyEqual(data["finished"], True)
2677             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2678             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2679             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2680             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2681             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2682             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2683             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2684             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2685             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2686             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2687         d.addCallback(_check_json)
2688         d.addCallback(self.get_operation_results, "124", "html")
2689         def _check_html(res):
2690             self.failUnless("Objects Checked: <span>10</span>" in res)
2691
2692             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2693             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2694             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2695
2696             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2697             self.failUnless("Repairs Successful: <span>0</span>" in res)
2698             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2699
2700             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2701             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2702             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2703         d.addCallback(_check_html)
2704         return d
2705
2706     def test_POST_FILEURL_bad_t(self):
2707         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2708                              "POST to file: bad t=bogus",
2709                              self.POST, self.public_url + "/foo/bar.txt",
2710                              t="bogus")
2711         return d
2712
2713     def test_POST_mkdir(self): # return value?
2714         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2715         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2716         d.addCallback(self.failUnlessNodeKeysAre, [])
2717         return d
2718
2719     def test_POST_mkdir_mdmf(self):
2720         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=mdmf")
2721         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2722         d.addCallback(lambda node:
2723             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2724         return d
2725
2726     def test_POST_mkdir_sdmf(self):
2727         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=sdmf")
2728         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2729         d.addCallback(lambda node:
2730             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2731         return d
2732
2733     def test_POST_mkdir_bad_mutable_type(self):
2734         return self.shouldHTTPError("test bad mutable type",
2735                                     400, "Bad Request", "Unknown type: foo",
2736                                     self.POST, self.public_url + \
2737                                     "/foo?t=mkdir&name=newdir&mutable-type=foo")
2738
2739     def test_POST_mkdir_initial_children(self):
2740         (newkids, caps) = self._create_initial_children()
2741         d = self.POST2(self.public_url +
2742                        "/foo?t=mkdir-with-children&name=newdir",
2743                        simplejson.dumps(newkids))
2744         d.addCallback(lambda res:
2745                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2747         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2748         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2749         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2750         return d
2751
2752     def test_POST_mkdir_initial_children_mdmf(self):
2753         (newkids, caps) = self._create_initial_children()
2754         d = self.POST2(self.public_url +
2755                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=mdmf",
2756                        simplejson.dumps(newkids))
2757         d.addCallback(lambda res:
2758                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2759         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2760         d.addCallback(lambda node:
2761             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2762         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2763         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2764                        caps['filecap1'])
2765         return d
2766
2767     # XXX: Duplication.
2768     def test_POST_mkdir_initial_children_sdmf(self):
2769         (newkids, caps) = self._create_initial_children()
2770         d = self.POST2(self.public_url +
2771                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=sdmf",
2772                        simplejson.dumps(newkids))
2773         d.addCallback(lambda res:
2774                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2775         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2776         d.addCallback(lambda node:
2777             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2778         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2779         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2780                        caps['filecap1'])
2781         return d
2782
2783     def test_POST_mkdir_initial_children_bad_mutable_type(self):
2784         (newkids, caps) = self._create_initial_children()
2785         return self.shouldHTTPError("test bad mutable type",
2786                                     400, "Bad Request", "Unknown type: foo",
2787                                     self.POST, self.public_url + \
2788                                     "/foo?t=mkdir-with-children&name=newdir&mutable-type=foo",
2789                                     simplejson.dumps(newkids))
2790
2791     def test_POST_mkdir_immutable(self):
2792         (newkids, caps) = self._create_immutable_children()
2793         d = self.POST2(self.public_url +
2794                        "/foo?t=mkdir-immutable&name=newdir",
2795                        simplejson.dumps(newkids))
2796         d.addCallback(lambda res:
2797                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2798         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2799         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2800         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2801         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2802         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2803         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2804         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2806         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2807         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2808         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2809         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2810         return d
2811
2812     def test_POST_mkdir_immutable_bad(self):
2813         (newkids, caps) = self._create_initial_children()
2814         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2815                              "400 Bad Request",
2816                              "needed to be immutable but was not",
2817                              self.POST2,
2818                              self.public_url +
2819                              "/foo?t=mkdir-immutable&name=newdir",
2820                              simplejson.dumps(newkids))
2821         return d
2822
2823     def test_POST_mkdir_2(self):
2824         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2825         d.addCallback(lambda res:
2826                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2827         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2828         d.addCallback(self.failUnlessNodeKeysAre, [])
2829         return d
2830
2831     def test_POST_mkdirs_2(self):
2832         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2833         d.addCallback(lambda res:
2834                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2835         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2836         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2837         d.addCallback(self.failUnlessNodeKeysAre, [])
2838         return d
2839
2840     def test_POST_mkdir_no_parentdir_noredirect(self):
2841         d = self.POST("/uri?t=mkdir")
2842         def _after_mkdir(res):
2843             uri.DirectoryURI.init_from_string(res)
2844         d.addCallback(_after_mkdir)
2845         return d
2846
2847     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2848         d = self.POST("/uri?t=mkdir&mutable-type=mdmf")
2849         def _after_mkdir(res):
2850             u = uri.from_string(res)
2851             # Check that this is an MDMF writecap
2852             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2853         d.addCallback(_after_mkdir)
2854         return d
2855
2856     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2857         d = self.POST("/uri?t=mkdir&mutable-type=sdmf")
2858         def _after_mkdir(res):
2859             u = uri.from_string(res)
2860             self.failUnlessIsInstance(u, uri.DirectoryURI)
2861         d.addCallback(_after_mkdir)
2862         return d
2863
2864     def test_POST_mkdir_no_parentdir_noredirect_bad_mutable_type(self):
2865         return self.shouldHTTPError("test bad mutable type",
2866                                     400, "Bad Request", "Unknown type: foo",
2867                                     self.POST, self.public_url + \
2868                                     "/uri?t=mkdir&mutable-type=foo")
2869
2870     def test_POST_mkdir_no_parentdir_noredirect2(self):
2871         # make sure form-based arguments (as on the welcome page) still work
2872         d = self.POST("/uri", t="mkdir")
2873         def _after_mkdir(res):
2874             uri.DirectoryURI.init_from_string(res)
2875         d.addCallback(_after_mkdir)
2876         d.addErrback(self.explain_web_error)
2877         return d
2878
2879     def test_POST_mkdir_no_parentdir_redirect(self):
2880         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2881         d.addBoth(self.shouldRedirect, None, statuscode='303')
2882         def _check_target(target):
2883             target = urllib.unquote(target)
2884             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2885         d.addCallback(_check_target)
2886         return d
2887
2888     def test_POST_mkdir_no_parentdir_redirect2(self):
2889         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2890         d.addBoth(self.shouldRedirect, None, statuscode='303')
2891         def _check_target(target):
2892             target = urllib.unquote(target)
2893             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2894         d.addCallback(_check_target)
2895         d.addErrback(self.explain_web_error)
2896         return d
2897
2898     def _make_readonly(self, u):
2899         ro_uri = uri.from_string(u).get_readonly()
2900         if ro_uri is None:
2901             return None
2902         return ro_uri.to_string()
2903
2904     def _create_initial_children(self):
2905         contents, n, filecap1 = self.makefile(12)
2906         md1 = {"metakey1": "metavalue1"}
2907         filecap2 = make_mutable_file_uri()
2908         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2909         filecap3 = node3.get_readonly_uri()
2910         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2911         dircap = DirectoryNode(node4, None, None).get_uri()
2912         mdmfcap = make_mutable_file_uri(mdmf=True)
2913         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2914         emptydircap = "URI:DIR2-LIT:"
2915         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2916                                                       "ro_uri": self._make_readonly(filecap1),
2917                                                       "metadata": md1, }],
2918                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2919                                                       "ro_uri": self._make_readonly(filecap2)}],
2920                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2921                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2922                                                       "ro_uri": unknown_rocap}],
2923                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2924                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2925                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2926                                                       "ro_uri": self._make_readonly(dircap)}],
2927                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2928                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2929                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2930                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2931                    }
2932         return newkids, {'filecap1': filecap1,
2933                          'filecap2': filecap2,
2934                          'filecap3': filecap3,
2935                          'unknown_rwcap': unknown_rwcap,
2936                          'unknown_rocap': unknown_rocap,
2937                          'unknown_immcap': unknown_immcap,
2938                          'dircap': dircap,
2939                          'litdircap': litdircap,
2940                          'emptydircap': emptydircap,
2941                          'mdmfcap': mdmfcap}
2942
2943     def _create_immutable_children(self):
2944         contents, n, filecap1 = self.makefile(12)
2945         md1 = {"metakey1": "metavalue1"}
2946         tnode = create_chk_filenode("immutable directory contents\n"*10)
2947         dnode = DirectoryNode(tnode, None, None)
2948         assert not dnode.is_mutable()
2949         immdircap = dnode.get_uri()
2950         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2951         emptydircap = "URI:DIR2-LIT:"
2952         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2953                                                       "metadata": md1, }],
2954                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2955                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2956                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2957                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2958                    }
2959         return newkids, {'filecap1': filecap1,
2960                          'unknown_immcap': unknown_immcap,
2961                          'immdircap': immdircap,
2962                          'litdircap': litdircap,
2963                          'emptydircap': emptydircap}
2964
2965     def test_POST_mkdir_no_parentdir_initial_children(self):
2966         (newkids, caps) = self._create_initial_children()
2967         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2968         def _after_mkdir(res):
2969             self.failUnless(res.startswith("URI:DIR"), res)
2970             n = self.s.create_node_from_uri(res)
2971             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2972             d2.addCallback(lambda ign:
2973                            self.failUnlessROChildURIIs(n, u"child-imm",
2974                                                        caps['filecap1']))
2975             d2.addCallback(lambda ign:
2976                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2977                                                        caps['filecap2']))
2978             d2.addCallback(lambda ign:
2979                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2980                                                        caps['filecap3']))
2981             d2.addCallback(lambda ign:
2982                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2983                                                        caps['unknown_rwcap']))
2984             d2.addCallback(lambda ign:
2985                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2986                                                        caps['unknown_rocap']))
2987             d2.addCallback(lambda ign:
2988                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2989                                                        caps['unknown_immcap']))
2990             d2.addCallback(lambda ign:
2991                            self.failUnlessRWChildURIIs(n, u"dirchild",
2992                                                        caps['dircap']))
2993             return d2
2994         d.addCallback(_after_mkdir)
2995         return d
2996
2997     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2998         # the regular /uri?t=mkdir operation is specified to ignore its body.
2999         # Only t=mkdir-with-children pays attention to it.
3000         (newkids, caps) = self._create_initial_children()
3001         d = self.shouldHTTPError("POST t=mkdir unexpected children",
3002                                  400, "Bad Request",
3003                                  "t=mkdir does not accept children=, "
3004                                  "try t=mkdir-with-children instead",
3005                                  self.POST2, "/uri?t=mkdir", # without children
3006                                  simplejson.dumps(newkids))
3007         return d
3008
3009     def test_POST_noparent_bad(self):
3010         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
3011                                  "/uri accepts only PUT, PUT?t=mkdir, "
3012                                  "POST?t=upload, and POST?t=mkdir",
3013                                  self.POST, "/uri?t=bogus")
3014         return d
3015
3016     def test_POST_mkdir_no_parentdir_immutable(self):
3017         (newkids, caps) = self._create_immutable_children()
3018         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3019         def _after_mkdir(res):
3020             self.failUnless(res.startswith("URI:DIR"), res)
3021             n = self.s.create_node_from_uri(res)
3022             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3023             d2.addCallback(lambda ign:
3024                            self.failUnlessROChildURIIs(n, u"child-imm",
3025                                                           caps['filecap1']))
3026             d2.addCallback(lambda ign:
3027                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3028                                                           caps['unknown_immcap']))
3029             d2.addCallback(lambda ign:
3030                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3031                                                           caps['immdircap']))
3032             d2.addCallback(lambda ign:
3033                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3034                                                           caps['litdircap']))
3035             d2.addCallback(lambda ign:
3036                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3037                                                           caps['emptydircap']))
3038             return d2
3039         d.addCallback(_after_mkdir)
3040         return d
3041
3042     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3043         (newkids, caps) = self._create_initial_children()
3044         d = self.shouldFail2(error.Error,
3045                              "test_POST_mkdir_no_parentdir_immutable_bad",
3046                              "400 Bad Request",
3047                              "needed to be immutable but was not",
3048                              self.POST2,
3049                              "/uri?t=mkdir-immutable",
3050                              simplejson.dumps(newkids))
3051         return d
3052
3053     def test_welcome_page_mkdir_button(self):
3054         # Fetch the welcome page.
3055         d = self.GET("/")
3056         def _after_get_welcome_page(res):
3057             MKDIR_BUTTON_RE = re.compile(
3058                 '<form action="([^"]*)" method="post".*?'
3059                 '<input type="hidden" name="t" value="([^"]*)" />'
3060                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3061                 '<input type="submit" value="Create a directory" />',
3062                 re.I)
3063             mo = MKDIR_BUTTON_RE.search(res)
3064             formaction = mo.group(1)
3065             formt = mo.group(2)
3066             formaname = mo.group(3)
3067             formavalue = mo.group(4)
3068             return (formaction, formt, formaname, formavalue)
3069         d.addCallback(_after_get_welcome_page)
3070         def _after_parse_form(res):
3071             (formaction, formt, formaname, formavalue) = res
3072             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3073         d.addCallback(_after_parse_form)
3074         d.addBoth(self.shouldRedirect, None, statuscode='303')
3075         return d
3076
3077     def test_POST_mkdir_replace(self): # return value?
3078         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3079         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3080         d.addCallback(self.failUnlessNodeKeysAre, [])
3081         return d
3082
3083     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3084         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3085         d.addBoth(self.shouldFail, error.Error,
3086                   "POST_mkdir_no_replace_queryarg",
3087                   "409 Conflict",
3088                   "There was already a child by that name, and you asked me "
3089                   "to not replace it")
3090         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3091         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3092         return d
3093
3094     def test_POST_mkdir_no_replace_field(self): # return value?
3095         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3096                       replace="false")
3097         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3098                   "409 Conflict",
3099                   "There was already a child by that name, and you asked me "
3100                   "to not replace it")
3101         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3102         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3103         return d
3104
3105     def test_POST_mkdir_whendone_field(self):
3106         d = self.POST(self.public_url + "/foo",
3107                       t="mkdir", name="newdir", when_done="/THERE")
3108         d.addBoth(self.shouldRedirect, "/THERE")
3109         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3110         d.addCallback(self.failUnlessNodeKeysAre, [])
3111         return d
3112
3113     def test_POST_mkdir_whendone_queryarg(self):
3114         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3115                       t="mkdir", name="newdir")
3116         d.addBoth(self.shouldRedirect, "/THERE")
3117         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3118         d.addCallback(self.failUnlessNodeKeysAre, [])
3119         return d
3120
3121     def test_POST_bad_t(self):
3122         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
3123                              "POST to a directory with bad t=BOGUS",
3124                              self.POST, self.public_url + "/foo", t="BOGUS")
3125         return d
3126
3127     def test_POST_set_children(self, command_name="set_children"):
3128         contents9, n9, newuri9 = self.makefile(9)
3129         contents10, n10, newuri10 = self.makefile(10)
3130         contents11, n11, newuri11 = self.makefile(11)
3131
3132         reqbody = """{
3133                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3134                                                 "size": 0,
3135                                                 "metadata": {
3136                                                   "ctime": 1002777696.7564139,
3137                                                   "mtime": 1002777696.7564139
3138                                                  }
3139                                                } ],
3140                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3141                                                 "size": 1,
3142                                                 "metadata": {
3143                                                   "ctime": 1002777696.7564139,
3144                                                   "mtime": 1002777696.7564139
3145                                                  }
3146                                                } ],
3147                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3148                                                 "size": 2,
3149                                                 "metadata": {
3150                                                   "ctime": 1002777696.7564139,
3151                                                   "mtime": 1002777696.7564139
3152                                                  }
3153                                                } ]
3154                     }""" % (newuri9, newuri10, newuri11)
3155
3156         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3157
3158         d = client.getPage(url, method="POST", postdata=reqbody)
3159         def _then(res):
3160             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3161             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3162             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3163
3164         d.addCallback(_then)
3165         d.addErrback(self.dump_error)
3166         return d
3167
3168     def test_POST_set_children_with_hyphen(self):
3169         return self.test_POST_set_children(command_name="set-children")
3170
3171     def test_POST_link_uri(self):
3172         contents, n, newuri = self.makefile(8)
3173         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3174         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3175         d.addCallback(lambda res:
3176                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3177                                                       contents))
3178         return d
3179
3180     def test_POST_link_uri_replace(self):
3181         contents, n, newuri = self.makefile(8)
3182         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3183         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3184         d.addCallback(lambda res:
3185                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3186                                                       contents))
3187         return d
3188
3189     def test_POST_link_uri_unknown_bad(self):
3190         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3191         d.addBoth(self.shouldFail, error.Error,
3192                   "POST_link_uri_unknown_bad",
3193                   "400 Bad Request",
3194                   "unknown cap in a write slot")
3195         return d
3196
3197     def test_POST_link_uri_unknown_ro_good(self):
3198         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3199         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3200         return d
3201
3202     def test_POST_link_uri_unknown_imm_good(self):
3203         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3204         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3205         return d
3206
3207     def test_POST_link_uri_no_replace_queryarg(self):
3208         contents, n, newuri = self.makefile(8)
3209         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3210                       name="bar.txt", uri=newuri)
3211         d.addBoth(self.shouldFail, error.Error,
3212                   "POST_link_uri_no_replace_queryarg",
3213                   "409 Conflict",
3214                   "There was already a child by that name, and you asked me "
3215                   "to not replace it")
3216         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3217         d.addCallback(self.failUnlessIsBarDotTxt)
3218         return d
3219
3220     def test_POST_link_uri_no_replace_field(self):
3221         contents, n, newuri = self.makefile(8)
3222         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3223                       name="bar.txt", uri=newuri)
3224         d.addBoth(self.shouldFail, error.Error,
3225                   "POST_link_uri_no_replace_field",
3226                   "409 Conflict",
3227                   "There was already a child by that name, and you asked me "
3228                   "to not replace it")
3229         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3230         d.addCallback(self.failUnlessIsBarDotTxt)
3231         return d
3232
3233     def test_POST_delete(self, command_name='delete'):
3234         d = self._foo_node.list()
3235         def _check_before(children):
3236             self.failUnless(u"bar.txt" in children)
3237         d.addCallback(_check_before)
3238         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3239         d.addCallback(lambda res: self._foo_node.list())
3240         def _check_after(children):
3241             self.failIf(u"bar.txt" in children)
3242         d.addCallback(_check_after)
3243         return d
3244
3245     def test_POST_unlink(self):
3246         return self.test_POST_delete(command_name='unlink')
3247
3248     def test_POST_rename_file(self):
3249         d = self.POST(self.public_url + "/foo", t="rename",
3250                       from_name="bar.txt", to_name='wibble.txt')
3251         d.addCallback(lambda res:
3252                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3253         d.addCallback(lambda res:
3254                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3255         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3256         d.addCallback(self.failUnlessIsBarDotTxt)
3257         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3258         d.addCallback(self.failUnlessIsBarJSON)
3259         return d
3260
3261     def test_POST_rename_file_redundant(self):
3262         d = self.POST(self.public_url + "/foo", t="rename",
3263                       from_name="bar.txt", to_name='bar.txt')
3264         d.addCallback(lambda res:
3265                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3266         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3267         d.addCallback(self.failUnlessIsBarDotTxt)
3268         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3269         d.addCallback(self.failUnlessIsBarJSON)
3270         return d
3271
3272     def test_POST_rename_file_replace(self):
3273         # rename a file and replace a directory with it
3274         d = self.POST(self.public_url + "/foo", t="rename",
3275                       from_name="bar.txt", to_name='empty')
3276         d.addCallback(lambda res:
3277                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3278         d.addCallback(lambda res:
3279                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3280         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3281         d.addCallback(self.failUnlessIsBarDotTxt)
3282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3283         d.addCallback(self.failUnlessIsBarJSON)
3284         return d
3285
3286     def test_POST_rename_file_no_replace_queryarg(self):
3287         # rename a file and replace a directory with it
3288         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3289                       from_name="bar.txt", to_name='empty')
3290         d.addBoth(self.shouldFail, error.Error,
3291                   "POST_rename_file_no_replace_queryarg",
3292                   "409 Conflict",
3293                   "There was already a child by that name, and you asked me "
3294                   "to not replace it")
3295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3296         d.addCallback(self.failUnlessIsEmptyJSON)
3297         return d
3298
3299     def test_POST_rename_file_no_replace_field(self):
3300         # rename a file and replace a directory with it
3301         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3302                       from_name="bar.txt", to_name='empty')
3303         d.addBoth(self.shouldFail, error.Error,
3304                   "POST_rename_file_no_replace_field",
3305                   "409 Conflict",
3306                   "There was already a child by that name, and you asked me "
3307                   "to not replace it")
3308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3309         d.addCallback(self.failUnlessIsEmptyJSON)
3310         return d
3311
3312     def failUnlessIsEmptyJSON(self, res):
3313         data = simplejson.loads(res)
3314         self.failUnlessEqual(data[0], "dirnode", data)
3315         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3316
3317     def test_POST_rename_file_slash_fail(self):
3318         d = self.POST(self.public_url + "/foo", t="rename",
3319                       from_name="bar.txt", to_name='kirk/spock.txt')
3320         d.addBoth(self.shouldFail, error.Error,
3321                   "test_POST_rename_file_slash_fail",
3322                   "400 Bad Request",
3323                   "to_name= may not contain a slash",
3324                   )
3325         d.addCallback(lambda res:
3326                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3327         return d
3328
3329     def test_POST_rename_dir(self):
3330         d = self.POST(self.public_url, t="rename",
3331                       from_name="foo", to_name='plunk')
3332         d.addCallback(lambda res:
3333                       self.failIfNodeHasChild(self.public_root, u"foo"))
3334         d.addCallback(lambda res:
3335                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3336         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3337         d.addCallback(self.failUnlessIsFooJSON)
3338         return d
3339
3340     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3341         """ If target is not None then the redirection has to go to target.  If
3342         statuscode is not None then the redirection has to be accomplished with
3343         that HTTP status code."""
3344         if not isinstance(res, failure.Failure):
3345             to_where = (target is None) and "somewhere" or ("to " + target)
3346             self.fail("%s: we were expecting to get redirected %s, not get an"
3347                       " actual page: %s" % (which, to_where, res))
3348         res.trap(error.PageRedirect)
3349         if statuscode is not None:
3350             self.failUnlessReallyEqual(res.value.status, statuscode,
3351                                        "%s: not a redirect" % which)
3352         if target is not None:
3353             # the PageRedirect does not seem to capture the uri= query arg
3354             # properly, so we can't check for it.
3355             realtarget = self.webish_url + target
3356             self.failUnlessReallyEqual(res.value.location, realtarget,
3357                                        "%s: wrong target" % which)
3358         return res.value.location
3359
3360     def test_GET_URI_form(self):
3361         base = "/uri?uri=%s" % self._bar_txt_uri
3362         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3363         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3364         d = self.GET(base)
3365         d.addBoth(self.shouldRedirect, targetbase)
3366         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3367         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3368         d.addCallback(lambda res: self.GET(base+"&t=json"))
3369         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3370         d.addCallback(self.log, "about to get file by uri")
3371         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3372         d.addCallback(self.failUnlessIsBarDotTxt)
3373         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3374         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3375                                            followRedirect=True))
3376         d.addCallback(self.failUnlessIsFooJSON)
3377         d.addCallback(self.log, "got dir by uri")
3378
3379         return d
3380
3381     def test_GET_URI_form_bad(self):
3382         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3383                              "400 Bad Request", "GET /uri requires uri=",
3384                              self.GET, "/uri")
3385         return d
3386
3387     def test_GET_rename_form(self):
3388         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3389                      followRedirect=True)
3390         def _check(res):
3391             self.failUnless('name="when_done" value="."' in res, res)
3392             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3393         d.addCallback(_check)
3394         return d
3395
3396     def log(self, res, msg):
3397         #print "MSG: %s  RES: %s" % (msg, res)
3398         log.msg(msg)
3399         return res
3400
3401     def test_GET_URI_URL(self):
3402         base = "/uri/%s" % self._bar_txt_uri
3403         d = self.GET(base)
3404         d.addCallback(self.failUnlessIsBarDotTxt)
3405         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3406         d.addCallback(self.failUnlessIsBarDotTxt)
3407         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3408         d.addCallback(self.failUnlessIsBarDotTxt)
3409         return d
3410
3411     def test_GET_URI_URL_dir(self):
3412         base = "/uri/%s?t=json" % self._foo_uri
3413         d = self.GET(base)
3414         d.addCallback(self.failUnlessIsFooJSON)
3415         return d
3416
3417     def test_GET_URI_URL_missing(self):
3418         base = "/uri/%s" % self._bad_file_uri
3419         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3420                                  http.GONE, None, "NotEnoughSharesError",
3421                                  self.GET, base)
3422         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3423         # here? we must arrange for a download to fail after target.open()
3424         # has been called, and then inspect the response to see that it is
3425         # shorter than we expected.
3426         return d
3427
3428     def test_PUT_DIRURL_uri(self):
3429         d = self.s.create_dirnode()
3430         def _made_dir(dn):
3431             new_uri = dn.get_uri()
3432             # replace /foo with a new (empty) directory
3433             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3434             d.addCallback(lambda res:
3435                           self.failUnlessReallyEqual(res.strip(), new_uri))
3436             d.addCallback(lambda res:
3437                           self.failUnlessRWChildURIIs(self.public_root,
3438                                                       u"foo",
3439                                                       new_uri))
3440             return d
3441         d.addCallback(_made_dir)
3442         return d
3443
3444     def test_PUT_DIRURL_uri_noreplace(self):
3445         d = self.s.create_dirnode()
3446         def _made_dir(dn):
3447             new_uri = dn.get_uri()
3448             # replace /foo with a new (empty) directory, but ask that
3449             # replace=false, so it should fail
3450             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3451                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3452                                  self.PUT,
3453                                  self.public_url + "/foo?t=uri&replace=false",
3454                                  new_uri)
3455             d.addCallback(lambda res:
3456                           self.failUnlessRWChildURIIs(self.public_root,
3457                                                       u"foo",
3458                                                       self._foo_uri))
3459             return d
3460         d.addCallback(_made_dir)
3461         return d
3462
3463     def test_PUT_DIRURL_bad_t(self):
3464         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3465                                  "400 Bad Request", "PUT to a directory",
3466                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
3467         d.addCallback(lambda res:
3468                       self.failUnlessRWChildURIIs(self.public_root,
3469                                                   u"foo",
3470                                                   self._foo_uri))
3471         return d
3472
3473     def test_PUT_NEWFILEURL_uri(self):
3474         contents, n, new_uri = self.makefile(8)
3475         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3476         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3477         d.addCallback(lambda res:
3478                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3479                                                       contents))
3480         return d
3481
3482     def test_PUT_NEWFILEURL_mdmf(self):
3483         new_contents = self.NEWFILE_CONTENTS * 300000
3484         d = self.PUT(self.public_url + \
3485                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
3486                      new_contents)
3487         d.addCallback(lambda ignored:
3488             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3489         def _got_json(json):
3490             data = simplejson.loads(json)
3491             data = data[1]
3492             self.failUnlessIn("mutable-type", data)
3493             self.failUnlessEqual(data['mutable-type'], "mdmf")
3494             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3495             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3496         d.addCallback(_got_json)
3497         return d
3498
3499     def test_PUT_NEWFILEURL_sdmf(self):
3500         new_contents = self.NEWFILE_CONTENTS * 300000
3501         d = self.PUT(self.public_url + \
3502                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
3503                      new_contents)
3504         d.addCallback(lambda ignored:
3505             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3506         def _got_json(json):
3507             data = simplejson.loads(json)
3508             data = data[1]
3509             self.failUnlessIn("mutable-type", data)
3510             self.failUnlessEqual(data['mutable-type'], "sdmf")
3511         d.addCallback(_got_json)
3512         return d
3513
3514     def test_PUT_NEWFILEURL_bad_mutable_type(self):
3515        new_contents = self.NEWFILE_CONTENTS * 300000
3516        return self.shouldHTTPError("test bad mutable type",
3517                                    400, "Bad Request", "Unknown type: foo",
3518                                    self.PUT, self.public_url + \
3519                                    "/foo/foo.txt?mutable=true&mutable-type=foo",
3520                                    new_contents)
3521
3522     def test_PUT_NEWFILEURL_uri_replace(self):
3523         contents, n, new_uri = self.makefile(8)
3524         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3525         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3526         d.addCallback(lambda res:
3527                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3528                                                       contents))
3529         return d
3530
3531     def test_PUT_NEWFILEURL_uri_no_replace(self):
3532         contents, n, new_uri = self.makefile(8)
3533         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3534         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
3535                   "409 Conflict",
3536                   "There was already a child by that name, and you asked me "
3537                   "to not replace it")
3538         return d
3539
3540     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3541         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3542         d.addBoth(self.shouldFail, error.Error,
3543                   "POST_put_uri_unknown_bad",
3544                   "400 Bad Request",
3545                   "unknown cap in a write slot")
3546         return d
3547
3548     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3549         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3550         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3551                       u"put-future-ro.txt")
3552         return d
3553
3554     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3555         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3556         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3557                       u"put-future-imm.txt")
3558         return d
3559
3560     def test_PUT_NEWFILE_URI(self):
3561         file_contents = "New file contents here\n"
3562         d = self.PUT("/uri", file_contents)
3563         def _check(uri):
3564             assert isinstance(uri, str), uri
3565             self.failUnless(uri in FakeCHKFileNode.all_contents)
3566             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3567                                        file_contents)
3568             return self.GET("/uri/%s" % uri)
3569         d.addCallback(_check)
3570         def _check2(res):
3571             self.failUnlessReallyEqual(res, file_contents)
3572         d.addCallback(_check2)
3573         return d
3574
3575     def test_PUT_NEWFILE_URI_not_mutable(self):
3576         file_contents = "New file contents here\n"
3577         d = self.PUT("/uri?mutable=false", file_contents)
3578         def _check(uri):
3579             assert isinstance(uri, str), uri
3580             self.failUnless(uri in FakeCHKFileNode.all_contents)
3581             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3582                                        file_contents)
3583             return self.GET("/uri/%s" % uri)
3584         d.addCallback(_check)
3585         def _check2(res):
3586             self.failUnlessReallyEqual(res, file_contents)
3587         d.addCallback(_check2)
3588         return d
3589
3590     def test_PUT_NEWFILE_URI_only_PUT(self):
3591         d = self.PUT("/uri?t=bogus", "")
3592         d.addBoth(self.shouldFail, error.Error,
3593                   "PUT_NEWFILE_URI_only_PUT",
3594                   "400 Bad Request",
3595                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3596         return d
3597
3598     def test_PUT_NEWFILE_URI_mutable(self):
3599         file_contents = "New file contents here\n"
3600         d = self.PUT("/uri?mutable=true", file_contents)
3601         def _check1(filecap):
3602             filecap = filecap.strip()
3603             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3604             self.filecap = filecap
3605             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3606             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3607             n = self.s.create_node_from_uri(filecap)
3608             return n.download_best_version()
3609         d.addCallback(_check1)
3610         def _check2(data):
3611             self.failUnlessReallyEqual(data, file_contents)
3612             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3613         d.addCallback(_check2)
3614         def _check3(res):
3615             self.failUnlessReallyEqual(res, file_contents)
3616         d.addCallback(_check3)
3617         return d
3618
3619     def test_PUT_mkdir(self):
3620         d = self.PUT("/uri?t=mkdir", "")
3621         def _check(uri):
3622             n = self.s.create_node_from_uri(uri.strip())
3623             d2 = self.failUnlessNodeKeysAre(n, [])
3624             d2.addCallback(lambda res:
3625                            self.GET("/uri/%s?t=json" % uri))
3626             return d2
3627         d.addCallback(_check)
3628         d.addCallback(self.failUnlessIsEmptyJSON)
3629         return d
3630
3631     def test_PUT_mkdir_mdmf(self):
3632         d = self.PUT("/uri?t=mkdir&mutable-type=mdmf", "")
3633         def _got(res):
3634             u = uri.from_string(res)
3635             # Check that this is an MDMF writecap
3636             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3637         d.addCallback(_got)
3638         return d
3639
3640     def test_PUT_mkdir_sdmf(self):
3641         d = self.PUT("/uri?t=mkdir&mutable-type=sdmf", "")
3642         def _got(res):
3643             u = uri.from_string(res)
3644             self.failUnlessIsInstance(u, uri.DirectoryURI)
3645         d.addCallback(_got)
3646         return d
3647
3648     def test_PUT_mkdir_bad_mutable_type(self):
3649         return self.shouldHTTPError("bad mutable type",
3650                                     400, "Bad Request", "Unknown type: foo",
3651                                     self.PUT, "/uri?t=mkdir&mutable-type=foo",
3652                                     "")
3653
3654     def test_POST_check(self):
3655         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3656         def _done(res):
3657             # this returns a string form of the results, which are probably
3658             # None since we're using fake filenodes.
3659             # TODO: verify that the check actually happened, by changing
3660             # FakeCHKFileNode to count how many times .check() has been
3661             # called.
3662             pass
3663         d.addCallback(_done)
3664         return d
3665
3666
3667     def test_PUT_update_at_offset(self):
3668         file_contents = "test file" * 100000 # about 900 KiB
3669         d = self.PUT("/uri?mutable=true", file_contents)
3670         def _then(filecap):
3671             self.filecap = filecap
3672             new_data = file_contents[:100]
3673             new = "replaced and so on"
3674             new_data += new
3675             new_data += file_contents[len(new_data):]
3676             assert len(new_data) == len(file_contents)
3677             self.new_data = new_data
3678         d.addCallback(_then)
3679         d.addCallback(lambda ignored:
3680             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3681                      "replaced and so on"))
3682         def _get_data(filecap):
3683             n = self.s.create_node_from_uri(filecap)
3684             return n.download_best_version()
3685         d.addCallback(_get_data)
3686         d.addCallback(lambda results:
3687             self.failUnlessEqual(results, self.new_data))
3688         # Now try appending things to the file
3689         d.addCallback(lambda ignored:
3690             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3691                      "puppies" * 100))
3692         d.addCallback(_get_data)
3693         d.addCallback(lambda results:
3694             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3695         # and try replacing the beginning of the file
3696         d.addCallback(lambda ignored:
3697             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3698         d.addCallback(_get_data)
3699         d.addCallback(lambda results:
3700             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3701         return d
3702
3703     def test_PUT_update_at_invalid_offset(self):
3704         file_contents = "test file" * 100000 # about 900 KiB
3705         d = self.PUT("/uri?mutable=true", file_contents)
3706         def _then(filecap):
3707             self.filecap = filecap
3708         d.addCallback(_then)
3709         # Negative offsets should cause an error.
3710         d.addCallback(lambda ignored:
3711             self.shouldHTTPError("test mutable invalid offset negative",
3712                                  400, "Bad Request",
3713                                  "Invalid offset",
3714                                  self.PUT,
3715                                  "/uri/%s?offset=-1" % self.filecap,
3716                                  "foo"))
3717         return d
3718
3719     def test_PUT_update_at_offset_immutable(self):
3720         file_contents = "Test file" * 100000
3721         d = self.PUT("/uri", file_contents)
3722         def _then(filecap):
3723             self.filecap = filecap
3724         d.addCallback(_then)
3725         d.addCallback(lambda ignored:
3726             self.shouldHTTPError("test immutable update",
3727                                  400, "Bad Request",
3728                                  "immutable",
3729                                  self.PUT,
3730                                  "/uri/%s?offset=50" % self.filecap,
3731                                  "foo"))
3732         return d
3733
3734
3735     def test_bad_method(self):
3736         url = self.webish_url + self.public_url + "/foo/bar.txt"
3737         d = self.shouldHTTPError("test_bad_method",
3738                                  501, "Not Implemented",
3739                                  "I don't know how to treat a BOGUS request.",
3740                                  client.getPage, url, method="BOGUS")
3741         return d
3742
3743     def test_short_url(self):
3744         url = self.webish_url + "/uri"
3745         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3746                                  "I don't know how to treat a DELETE request.",
3747                                  client.getPage, url, method="DELETE")
3748         return d
3749
3750     def test_ophandle_bad(self):
3751         url = self.webish_url + "/operations/bogus?t=status"
3752         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3753                                  "unknown/expired handle 'bogus'",
3754                                  client.getPage, url)
3755         return d
3756
3757     def test_ophandle_cancel(self):
3758         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3759                       followRedirect=True)
3760         d.addCallback(lambda ignored:
3761                       self.GET("/operations/128?t=status&output=JSON"))
3762         def _check1(res):
3763             data = simplejson.loads(res)
3764             self.failUnless("finished" in data, res)
3765             monitor = self.ws.root.child_operations.handles["128"][0]
3766             d = self.POST("/operations/128?t=cancel&output=JSON")
3767             def _check2(res):
3768                 data = simplejson.loads(res)
3769                 self.failUnless("finished" in data, res)
3770                 # t=cancel causes the handle to be forgotten
3771                 self.failUnless(monitor.is_cancelled())
3772             d.addCallback(_check2)
3773             return d
3774         d.addCallback(_check1)
3775         d.addCallback(lambda ignored:
3776                       self.shouldHTTPError("test_ophandle_cancel",
3777                                            404, "404 Not Found",
3778                                            "unknown/expired handle '128'",
3779                                            self.GET,
3780                                            "/operations/128?t=status&output=JSON"))
3781         return d
3782
3783     def test_ophandle_retainfor(self):
3784         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3785                       followRedirect=True)
3786         d.addCallback(lambda ignored:
3787                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3788         def _check1(res):
3789             data = simplejson.loads(res)
3790             self.failUnless("finished" in data, res)
3791         d.addCallback(_check1)
3792         # the retain-for=0 will cause the handle to be expired very soon
3793         d.addCallback(lambda ign:
3794             self.clock.advance(2.0))
3795         d.addCallback(lambda ignored:
3796                       self.shouldHTTPError("test_ophandle_retainfor",
3797                                            404, "404 Not Found",
3798                                            "unknown/expired handle '129'",
3799                                            self.GET,
3800                                            "/operations/129?t=status&output=JSON"))
3801         return d
3802
3803     def test_ophandle_release_after_complete(self):
3804         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3805                       followRedirect=True)
3806         d.addCallback(self.wait_for_operation, "130")
3807         d.addCallback(lambda ignored:
3808                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3809         # the release-after-complete=true will cause the handle to be expired
3810         d.addCallback(lambda ignored:
3811                       self.shouldHTTPError("test_ophandle_release_after_complete",
3812                                            404, "404 Not Found",
3813                                            "unknown/expired handle '130'",
3814                                            self.GET,
3815                                            "/operations/130?t=status&output=JSON"))
3816         return d
3817
3818     def test_uncollected_ophandle_expiration(self):
3819         # uncollected ophandles should expire after 4 days
3820         def _make_uncollected_ophandle(ophandle):
3821             d = self.POST(self.public_url +
3822                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3823                           followRedirect=False)
3824             # When we start the operation, the webapi server will want
3825             # to redirect us to the page for the ophandle, so we get
3826             # confirmation that the operation has started. If the
3827             # manifest operation has finished by the time we get there,
3828             # following that redirect (by setting followRedirect=True
3829             # above) has the side effect of collecting the ophandle that
3830             # we've just created, which means that we can't use the
3831             # ophandle to test the uncollected timeout anymore. So,
3832             # instead, catch the 302 here and don't follow it.
3833             d.addBoth(self.should302, "uncollected_ophandle_creation")
3834             return d
3835         # Create an ophandle, don't collect it, then advance the clock by
3836         # 4 days - 1 second and make sure that the ophandle is still there.
3837         d = _make_uncollected_ophandle(131)
3838         d.addCallback(lambda ign:
3839             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3840         d.addCallback(lambda ign:
3841             self.GET("/operations/131?t=status&output=JSON"))
3842         def _check1(res):
3843             data = simplejson.loads(res)
3844             self.failUnless("finished" in data, res)
3845         d.addCallback(_check1)
3846         # Create an ophandle, don't collect it, then try to collect it
3847         # after 4 days. It should be gone.
3848         d.addCallback(lambda ign:
3849             _make_uncollected_ophandle(132))
3850         d.addCallback(lambda ign:
3851             self.clock.advance(96*60*60))
3852         d.addCallback(lambda ign:
3853             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3854                                  404, "404 Not Found",
3855                                  "unknown/expired handle '132'",
3856                                  self.GET,
3857                                  "/operations/132?t=status&output=JSON"))
3858         return d
3859
3860     def test_collected_ophandle_expiration(self):
3861         # collected ophandles should expire after 1 day
3862         def _make_collected_ophandle(ophandle):
3863             d = self.POST(self.public_url +
3864                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3865                           followRedirect=True)
3866             # By following the initial redirect, we collect the ophandle
3867             # we've just created.
3868             return d
3869         # Create a collected ophandle, then collect it after 23 hours
3870         # and 59 seconds to make sure that it is still there.
3871         d = _make_collected_ophandle(133)
3872         d.addCallback(lambda ign:
3873             self.clock.advance((24*60*60) - 1))
3874         d.addCallback(lambda ign:
3875             self.GET("/operations/133?t=status&output=JSON"))
3876         def _check1(res):
3877             data = simplejson.loads(res)
3878             self.failUnless("finished" in data, res)
3879         d.addCallback(_check1)
3880         # Create another uncollected ophandle, then try to collect it
3881         # after 24 hours to make sure that it is gone.
3882         d.addCallback(lambda ign:
3883             _make_collected_ophandle(134))
3884         d.addCallback(lambda ign:
3885             self.clock.advance(24*60*60))
3886         d.addCallback(lambda ign:
3887             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3888                                  404, "404 Not Found",
3889                                  "unknown/expired handle '134'",
3890                                  self.GET,
3891                                  "/operations/134?t=status&output=JSON"))
3892         return d
3893
3894     def test_incident(self):
3895         d = self.POST("/report_incident", details="eek")
3896         def _done(res):
3897             self.failUnless("Thank you for your report!" in res, res)
3898         d.addCallback(_done)
3899         return d
3900
3901     def test_static(self):
3902         webdir = os.path.join(self.staticdir, "subdir")
3903         fileutil.make_dirs(webdir)
3904         f = open(os.path.join(webdir, "hello.txt"), "wb")
3905         f.write("hello")
3906         f.close()
3907
3908         d = self.GET("/static/subdir/hello.txt")
3909         def _check(res):
3910             self.failUnlessReallyEqual(res, "hello")
3911         d.addCallback(_check)
3912         return d
3913
3914
3915 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3916     def test_load_file(self):
3917         # This will raise an exception unless a well-formed XML file is found under that name.
3918         common.getxmlfile('directory.xhtml').load()
3919
3920     def test_parse_replace_arg(self):
3921         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3922         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3923         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3924                                    "only-files")
3925         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3926                         common.parse_replace_arg, "only_fles")
3927
3928     def test_abbreviate_time(self):
3929         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3930         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3931         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3932         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3933         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3934         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3935
3936     def test_compute_rate(self):
3937         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3938         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3939         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3940         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3941         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3942         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3943         self.shouldFail(AssertionError, "test_compute_rate", "",
3944                         common.compute_rate, -100, 10)
3945         self.shouldFail(AssertionError, "test_compute_rate", "",
3946                         common.compute_rate, 100, -10)
3947
3948         # Sanity check
3949         rate = common.compute_rate(10*1000*1000, 1)
3950         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3951
3952     def test_abbreviate_rate(self):
3953         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3954         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3955         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3956         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3957
3958     def test_abbreviate_size(self):
3959         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3960         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3961         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3962         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3963         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3964
3965     def test_plural(self):
3966         def convert(s):
3967             return "%d second%s" % (s, status.plural(s))
3968         self.failUnlessReallyEqual(convert(0), "0 seconds")
3969         self.failUnlessReallyEqual(convert(1), "1 second")
3970         self.failUnlessReallyEqual(convert(2), "2 seconds")
3971         def convert2(s):
3972             return "has share%s: %s" % (status.plural(s), ",".join(s))
3973         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3974         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3975         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3976
3977
3978 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3979
3980     def CHECK(self, ign, which, args, clientnum=0):
3981         fileurl = self.fileurls[which]
3982         url = fileurl + "?" + args
3983         return self.GET(url, method="POST", clientnum=clientnum)
3984
3985     def test_filecheck(self):
3986         self.basedir = "web/Grid/filecheck"
3987         self.set_up_grid()
3988         c0 = self.g.clients[0]
3989         self.uris = {}
3990         DATA = "data" * 100
3991         d = c0.upload(upload.Data(DATA, convergence=""))
3992         def _stash_uri(ur, which):
3993             self.uris[which] = ur.uri
3994         d.addCallback(_stash_uri, "good")
3995         d.addCallback(lambda ign:
3996                       c0.upload(upload.Data(DATA+"1", convergence="")))
3997         d.addCallback(_stash_uri, "sick")
3998         d.addCallback(lambda ign:
3999                       c0.upload(upload.Data(DATA+"2", convergence="")))
4000         d.addCallback(_stash_uri, "dead")
4001         def _stash_mutable_uri(n, which):
4002             self.uris[which] = n.get_uri()
4003             assert isinstance(self.uris[which], str)
4004         d.addCallback(lambda ign:
4005             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4006         d.addCallback(_stash_mutable_uri, "corrupt")
4007         d.addCallback(lambda ign:
4008                       c0.upload(upload.Data("literal", convergence="")))
4009         d.addCallback(_stash_uri, "small")
4010         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4011         d.addCallback(_stash_mutable_uri, "smalldir")
4012
4013         def _compute_fileurls(ignored):
4014             self.fileurls = {}
4015             for which in self.uris:
4016                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4017         d.addCallback(_compute_fileurls)
4018
4019         def _clobber_shares(ignored):
4020             good_shares = self.find_uri_shares(self.uris["good"])
4021             self.failUnlessReallyEqual(len(good_shares), 10)
4022             sick_shares = self.find_uri_shares(self.uris["sick"])
4023             os.unlink(sick_shares[0][2])
4024             dead_shares = self.find_uri_shares(self.uris["dead"])
4025             for i in range(1, 10):
4026                 os.unlink(dead_shares[i][2])
4027             c_shares = self.find_uri_shares(self.uris["corrupt"])
4028             cso = CorruptShareOptions()
4029             cso.stdout = StringIO()
4030             cso.parseOptions([c_shares[0][2]])
4031             corrupt_share(cso)
4032         d.addCallback(_clobber_shares)
4033
4034         d.addCallback(self.CHECK, "good", "t=check")
4035         def _got_html_good(res):
4036             self.failUnless("Healthy" in res, res)
4037             self.failIf("Not Healthy" in res, res)
4038         d.addCallback(_got_html_good)
4039         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4040         def _got_html_good_return_to(res):
4041             self.failUnless("Healthy" in res, res)
4042             self.failIf("Not Healthy" in res, res)
4043             self.failUnless('<a href="somewhere">Return to file'
4044                             in res, res)
4045         d.addCallback(_got_html_good_return_to)
4046         d.addCallback(self.CHECK, "good", "t=check&output=json")
4047         def _got_json_good(res):
4048             r = simplejson.loads(res)
4049             self.failUnlessEqual(r["summary"], "Healthy")
4050             self.failUnless(r["results"]["healthy"])
4051             self.failIf(r["results"]["needs-rebalancing"])
4052             self.failUnless(r["results"]["recoverable"])
4053         d.addCallback(_got_json_good)
4054
4055         d.addCallback(self.CHECK, "small", "t=check")
4056         def _got_html_small(res):
4057             self.failUnless("Literal files are always healthy" in res, res)
4058             self.failIf("Not Healthy" in res, res)
4059         d.addCallback(_got_html_small)
4060         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4061         def _got_html_small_return_to(res):
4062             self.failUnless("Literal files are always healthy" in res, res)
4063             self.failIf("Not Healthy" in res, res)
4064             self.failUnless('<a href="somewhere">Return to file'
4065                             in res, res)
4066         d.addCallback(_got_html_small_return_to)
4067         d.addCallback(self.CHECK, "small", "t=check&output=json")
4068         def _got_json_small(res):
4069             r = simplejson.loads(res)
4070             self.failUnlessEqual(r["storage-index"], "")
4071             self.failUnless(r["results"]["healthy"])
4072         d.addCallback(_got_json_small)
4073
4074         d.addCallback(self.CHECK, "smalldir", "t=check")
4075         def _got_html_smalldir(res):
4076             self.failUnless("Literal files are always healthy" in res, res)
4077             self.failIf("Not Healthy" in res, res)
4078         d.addCallback(_got_html_smalldir)
4079         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4080         def _got_json_smalldir(res):
4081             r = simplejson.loads(res)
4082             self.failUnlessEqual(r["storage-index"], "")
4083             self.failUnless(r["results"]["healthy"])
4084         d.addCallback(_got_json_smalldir)
4085
4086         d.addCallback(self.CHECK, "sick", "t=check")
4087         def _got_html_sick(res):
4088             self.failUnless("Not Healthy" in res, res)
4089         d.addCallback(_got_html_sick)
4090         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4091         def _got_json_sick(res):
4092             r = simplejson.loads(res)
4093             self.failUnlessEqual(r["summary"],
4094                                  "Not Healthy: 9 shares (enc 3-of-10)")
4095             self.failIf(r["results"]["healthy"])
4096             self.failIf(r["results"]["needs-rebalancing"])
4097             self.failUnless(r["results"]["recoverable"])
4098         d.addCallback(_got_json_sick)
4099
4100         d.addCallback(self.CHECK, "dead", "t=check")
4101         def _got_html_dead(res):
4102             self.failUnless("Not Healthy" in res, res)
4103         d.addCallback(_got_html_dead)
4104         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4105         def _got_json_dead(res):
4106             r = simplejson.loads(res)
4107             self.failUnlessEqual(r["summary"],
4108                                  "Not Healthy: 1 shares (enc 3-of-10)")
4109             self.failIf(r["results"]["healthy"])
4110             self.failIf(r["results"]["needs-rebalancing"])
4111             self.failIf(r["results"]["recoverable"])
4112         d.addCallback(_got_json_dead)
4113
4114         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4115         def _got_html_corrupt(res):
4116             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4117         d.addCallback(_got_html_corrupt)
4118         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4119         def _got_json_corrupt(res):
4120             r = simplejson.loads(res)
4121             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4122                             r["summary"])
4123             self.failIf(r["results"]["healthy"])
4124             self.failUnless(r["results"]["recoverable"])
4125             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4126             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4127         d.addCallback(_got_json_corrupt)
4128
4129         d.addErrback(self.explain_web_error)
4130         return d
4131
4132     def test_repair_html(self):
4133         self.basedir = "web/Grid/repair_html"
4134         self.set_up_grid()
4135         c0 = self.g.clients[0]
4136         self.uris = {}
4137         DATA = "data" * 100
4138         d = c0.upload(upload.Data(DATA, convergence=""))
4139         def _stash_uri(ur, which):
4140             self.uris[which] = ur.uri
4141         d.addCallback(_stash_uri, "good")
4142         d.addCallback(lambda ign:
4143                       c0.upload(upload.Data(DATA+"1", convergence="")))
4144         d.addCallback(_stash_uri, "sick")
4145         d.addCallback(lambda ign:
4146                       c0.upload(upload.Data(DATA+"2", convergence="")))
4147         d.addCallback(_stash_uri, "dead")
4148         def _stash_mutable_uri(n, which):
4149             self.uris[which] = n.get_uri()
4150             assert isinstance(self.uris[which], str)
4151         d.addCallback(lambda ign:
4152             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4153         d.addCallback(_stash_mutable_uri, "corrupt")
4154
4155         def _compute_fileurls(ignored):
4156             self.fileurls = {}
4157             for which in self.uris:
4158                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4159         d.addCallback(_compute_fileurls)
4160
4161         def _clobber_shares(ignored):
4162             good_shares = self.find_uri_shares(self.uris["good"])
4163             self.failUnlessReallyEqual(len(good_shares), 10)
4164             sick_shares = self.find_uri_shares(self.uris["sick"])
4165             os.unlink(sick_shares[0][2])
4166             dead_shares = self.find_uri_shares(self.uris["dead"])
4167             for i in range(1, 10):
4168                 os.unlink(dead_shares[i][2])
4169             c_shares = self.find_uri_shares(self.uris["corrupt"])
4170             cso = CorruptShareOptions()
4171             cso.stdout = StringIO()
4172             cso.parseOptions([c_shares[0][2]])
4173             corrupt_share(cso)
4174         d.addCallback(_clobber_shares)
4175
4176         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4177         def _got_html_good(res):
4178             self.failUnless("Healthy" in res, res)
4179             self.failIf("Not Healthy" in res, res)
4180             self.failUnless("No repair necessary" in res, res)
4181         d.addCallback(_got_html_good)
4182
4183         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4184         def _got_html_sick(res):
4185             self.failUnless("Healthy : healthy" in res, res)
4186             self.failIf("Not Healthy" in res, res)
4187             self.failUnless("Repair successful" in res, res)
4188         d.addCallback(_got_html_sick)
4189
4190         # repair of a dead file will fail, of course, but it isn't yet
4191         # clear how this should be reported. Right now it shows up as
4192         # a "410 Gone".
4193         #
4194         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4195         #def _got_html_dead(res):
4196         #    print res
4197         #    self.failUnless("Healthy : healthy" in res, res)
4198         #    self.failIf("Not Healthy" in res, res)
4199         #    self.failUnless("No repair necessary" in res, res)
4200         #d.addCallback(_got_html_dead)
4201
4202         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4203         def _got_html_corrupt(res):
4204             self.failUnless("Healthy : Healthy" in res, res)
4205             self.failIf("Not Healthy" in res, res)
4206             self.failUnless("Repair successful" in res, res)
4207         d.addCallback(_got_html_corrupt)
4208
4209         d.addErrback(self.explain_web_error)
4210         return d
4211
4212     def test_repair_json(self):
4213         self.basedir = "web/Grid/repair_json"
4214         self.set_up_grid()
4215         c0 = self.g.clients[0]
4216         self.uris = {}
4217         DATA = "data" * 100
4218         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4219         def _stash_uri(ur, which):
4220             self.uris[which] = ur.uri
4221         d.addCallback(_stash_uri, "sick")
4222
4223         def _compute_fileurls(ignored):
4224             self.fileurls = {}
4225             for which in self.uris:
4226                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4227         d.addCallback(_compute_fileurls)
4228
4229         def _clobber_shares(ignored):
4230             sick_shares = self.find_uri_shares(self.uris["sick"])
4231             os.unlink(sick_shares[0][2])
4232         d.addCallback(_clobber_shares)
4233
4234         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4235         def _got_json_sick(res):
4236             r = simplejson.loads(res)
4237             self.failUnlessReallyEqual(r["repair-attempted"], True)
4238             self.failUnlessReallyEqual(r["repair-successful"], True)
4239             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4240                                  "Not Healthy: 9 shares (enc 3-of-10)")
4241             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4242             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4243             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4244         d.addCallback(_got_json_sick)
4245
4246         d.addErrback(self.explain_web_error)
4247         return d
4248
4249     def test_unknown(self, immutable=False):
4250         self.basedir = "web/Grid/unknown"
4251         if immutable:
4252             self.basedir = "web/Grid/unknown-immutable"
4253
4254         self.set_up_grid()
4255         c0 = self.g.clients[0]
4256         self.uris = {}
4257         self.fileurls = {}
4258
4259         # the future cap format may contain slashes, which must be tolerated
4260         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4261                                                            safe="")
4262
4263         if immutable:
4264             name = u"future-imm"
4265             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4266             d = c0.create_immutable_dirnode({name: (future_node, {})})
4267         else:
4268             name = u"future"
4269             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4270             d = c0.create_dirnode()
4271
4272         def _stash_root_and_create_file(n):
4273             self.rootnode = n
4274             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4275             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4276             if not immutable:
4277                 return self.rootnode.set_node(name, future_node)
4278         d.addCallback(_stash_root_and_create_file)
4279
4280         # make sure directory listing tolerates unknown nodes
4281         d.addCallback(lambda ign: self.GET(self.rooturl))
4282         def _check_directory_html(res, expected_type_suffix):
4283             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4284                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4285                                  re.DOTALL)
4286             self.failUnless(re.search(pattern, res), res)
4287             # find the More Info link for name, should be relative
4288             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4289             info_url = mo.group(1)
4290             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4291         if immutable:
4292             d.addCallback(_check_directory_html, "-IMM")
4293         else:
4294             d.addCallback(_check_directory_html, "")
4295
4296         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4297         def _check_directory_json(res, expect_rw_uri):
4298             data = simplejson.loads(res)
4299             self.failUnlessEqual(data[0], "dirnode")
4300             f = data[1]["children"][name]
4301             self.failUnlessEqual(f[0], "unknown")
4302             if expect_rw_uri:
4303                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4304             else:
4305                 self.failIfIn("rw_uri", f[1])
4306             if immutable:
4307                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4308             else:
4309                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4310             self.failUnless("metadata" in f[1])
4311         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4312
4313         def _check_info(res, expect_rw_uri, expect_ro_uri):
4314             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4315             if expect_rw_uri:
4316                 self.failUnlessIn(unknown_rwcap, res)
4317             if expect_ro_uri:
4318                 if immutable:
4319                     self.failUnlessIn(unknown_immcap, res)
4320                 else:
4321                     self.failUnlessIn(unknown_rocap, res)
4322             else:
4323                 self.failIfIn(unknown_rocap, res)
4324             self.failIfIn("Raw data as", res)
4325             self.failIfIn("Directory writecap", res)
4326             self.failIfIn("Checker Operations", res)
4327             self.failIfIn("Mutable File Operations", res)
4328             self.failIfIn("Directory Operations", res)
4329
4330         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4331         # why they fail. Possibly related to ticket #922.
4332
4333         d.addCallback(lambda ign: self.GET(expected_info_url))
4334         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4335         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4336         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4337
4338         def _check_json(res, expect_rw_uri):
4339             data = simplejson.loads(res)
4340             self.failUnlessEqual(data[0], "unknown")
4341             if expect_rw_uri:
4342                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4343             else:
4344                 self.failIfIn("rw_uri", data[1])
4345
4346             if immutable:
4347                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4348                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4349             elif expect_rw_uri:
4350                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4351                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4352             else:
4353                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4354                 self.failIf("mutable" in data[1], data[1])
4355
4356             # TODO: check metadata contents
4357             self.failUnless("metadata" in data[1])
4358
4359         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4360         d.addCallback(_check_json, expect_rw_uri=not immutable)
4361
4362         # and make sure that a read-only version of the directory can be
4363         # rendered too. This version will not have unknown_rwcap, whether
4364         # or not future_node was immutable.
4365         d.addCallback(lambda ign: self.GET(self.rourl))
4366         if immutable:
4367             d.addCallback(_check_directory_html, "-IMM")
4368         else:
4369             d.addCallback(_check_directory_html, "-RO")
4370
4371         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4372         d.addCallback(_check_directory_json, expect_rw_uri=False)
4373
4374         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4375         d.addCallback(_check_json, expect_rw_uri=False)
4376
4377         # TODO: check that getting t=info from the Info link in the ro directory
4378         # works, and does not include the writecap URI.
4379         return d
4380
4381     def test_immutable_unknown(self):
4382         return self.test_unknown(immutable=True)
4383
4384     def test_mutant_dirnodes_are_omitted(self):
4385         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4386
4387         self.set_up_grid()
4388         c = self.g.clients[0]
4389         nm = c.nodemaker
4390         self.uris = {}
4391         self.fileurls = {}
4392
4393         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4394         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4395         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4396
4397         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4398         # test the dirnode and web layers separately.
4399
4400         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4401         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4402         # When the directory is read, the mutants should be silently disposed of, leaving
4403         # their lonely sibling.
4404         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4405         # because immutable directories don't have a writecap and therefore that field
4406         # isn't (and can't be) decrypted.
4407         # TODO: The field still exists in the netstring. Technically we should check what
4408         # happens if something is put there (_unpack_contents should raise ValueError),
4409         # but that can wait.
4410
4411         lonely_child = nm.create_from_cap(lonely_uri)
4412         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4413         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4414
4415         def _by_hook_or_by_crook():
4416             return True
4417         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4418             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4419
4420         mutant_write_in_ro_child.get_write_uri    = lambda: None
4421         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4422
4423         kids = {u"lonely":      (lonely_child, {}),
4424                 u"ro":          (mutant_ro_child, {}),
4425                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4426                 }
4427         d = c.create_immutable_dirnode(kids)
4428
4429         def _created(dn):
4430             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4431             self.failIf(dn.is_mutable())
4432             self.failUnless(dn.is_readonly())
4433             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4434             self.failIf(hasattr(dn._node, 'get_writekey'))
4435             rep = str(dn)
4436             self.failUnless("RO-IMM" in rep)
4437             cap = dn.get_cap()
4438             self.failUnlessIn("CHK", cap.to_string())
4439             self.cap = cap
4440             self.rootnode = dn
4441             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4442             return download_to_data(dn._node)
4443         d.addCallback(_created)
4444
4445         def _check_data(data):
4446             # Decode the netstring representation of the directory to check that all children
4447             # are present. This is a bit of an abstraction violation, but there's not really
4448             # any other way to do it given that the real DirectoryNode._unpack_contents would
4449             # strip the mutant children out (which is what we're trying to test, later).
4450             position = 0
4451             numkids = 0
4452             while position < len(data):
4453                 entries, position = split_netstring(data, 1, position)
4454                 entry = entries[0]
4455                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4456                 name = name_utf8.decode("utf-8")
4457                 self.failUnless(rwcapdata == "")
4458                 self.failUnless(name in kids)
4459                 (expected_child, ign) = kids[name]
4460                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4461                 numkids += 1
4462
4463             self.failUnlessReallyEqual(numkids, 3)
4464             return self.rootnode.list()
4465         d.addCallback(_check_data)
4466
4467         # Now when we use the real directory listing code, the mutants should be absent.
4468         def _check_kids(children):
4469             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4470             lonely_node, lonely_metadata = children[u"lonely"]
4471
4472             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4473             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4474         d.addCallback(_check_kids)
4475
4476         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4477         d.addCallback(lambda n: n.list())
4478         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4479
4480         # Make sure the lonely child can be listed in HTML...
4481         d.addCallback(lambda ign: self.GET(self.rooturl))
4482         def _check_html(res):
4483             self.failIfIn("URI:SSK", res)
4484             get_lonely = "".join([r'<td>FILE</td>',
4485                                   r'\s+<td>',
4486                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4487                                   r'</td>',
4488                                   r'\s+<td align="right">%d</td>' % len("one"),
4489                                   ])
4490             self.failUnless(re.search(get_lonely, res), res)
4491
4492             # find the More Info link for name, should be relative
4493             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4494             info_url = mo.group(1)
4495             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4496         d.addCallback(_check_html)
4497
4498         # ... and in JSON.
4499         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4500         def _check_json(res):
4501             data = simplejson.loads(res)
4502             self.failUnlessEqual(data[0], "dirnode")
4503             listed_children = data[1]["children"]
4504             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4505             ll_type, ll_data = listed_children[u"lonely"]
4506             self.failUnlessEqual(ll_type, "filenode")
4507             self.failIf("rw_uri" in ll_data)
4508             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4509         d.addCallback(_check_json)
4510         return d
4511
4512     def test_deep_check(self):
4513         self.basedir = "web/Grid/deep_check"
4514         self.set_up_grid()
4515         c0 = self.g.clients[0]
4516         self.uris = {}
4517         self.fileurls = {}
4518         DATA = "data" * 100
4519         d = c0.create_dirnode()
4520         def _stash_root_and_create_file(n):
4521             self.rootnode = n
4522             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4523             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4524         d.addCallback(_stash_root_and_create_file)
4525         def _stash_uri(fn, which):
4526             self.uris[which] = fn.get_uri()
4527             return fn
4528         d.addCallback(_stash_uri, "good")
4529         d.addCallback(lambda ign:
4530                       self.rootnode.add_file(u"small",
4531                                              upload.Data("literal",
4532                                                         convergence="")))
4533         d.addCallback(_stash_uri, "small")
4534         d.addCallback(lambda ign:
4535                       self.rootnode.add_file(u"sick",
4536                                              upload.Data(DATA+"1",
4537                                                         convergence="")))
4538         d.addCallback(_stash_uri, "sick")
4539
4540         # this tests that deep-check and stream-manifest will ignore
4541         # UnknownNode instances. Hopefully this will also cover deep-stats.
4542         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4543         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4544
4545         def _clobber_shares(ignored):
4546             self.delete_shares_numbered(self.uris["sick"], [0,1])
4547         d.addCallback(_clobber_shares)
4548
4549         # root
4550         # root/good
4551         # root/small
4552         # root/sick
4553         # root/future
4554
4555         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4556         def _done(res):
4557             try:
4558                 units = [simplejson.loads(line)
4559                          for line in res.splitlines()
4560                          if line]
4561             except ValueError:
4562                 print "response is:", res
4563                 print "undecodeable line was '%s'" % line
4564                 raise
4565             self.failUnlessReallyEqual(len(units), 5+1)
4566             # should be parent-first
4567             u0 = units[0]
4568             self.failUnlessEqual(u0["path"], [])
4569             self.failUnlessEqual(u0["type"], "directory")
4570             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4571             u0cr = u0["check-results"]
4572             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4573
4574             ugood = [u for u in units
4575                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4576             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4577             ugoodcr = ugood["check-results"]
4578             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4579
4580             stats = units[-1]
4581             self.failUnlessEqual(stats["type"], "stats")
4582             s = stats["stats"]
4583             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4584             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4585             self.failUnlessReallyEqual(s["count-directories"], 1)
4586             self.failUnlessReallyEqual(s["count-unknown"], 1)
4587         d.addCallback(_done)
4588
4589         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4590         def _check_manifest(res):
4591             self.failUnless(res.endswith("\n"))
4592             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4593             self.failUnlessReallyEqual(len(units), 5+1)
4594             self.failUnlessEqual(units[-1]["type"], "stats")
4595             first = units[0]
4596             self.failUnlessEqual(first["path"], [])
4597             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4598             self.failUnlessEqual(first["type"], "directory")
4599             stats = units[-1]["stats"]
4600             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4601             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4602             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4603             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4604             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4605         d.addCallback(_check_manifest)
4606
4607         # now add root/subdir and root/subdir/grandchild, then make subdir
4608         # unrecoverable, then see what happens
4609
4610         d.addCallback(lambda ign:
4611                       self.rootnode.create_subdirectory(u"subdir"))
4612         d.addCallback(_stash_uri, "subdir")
4613         d.addCallback(lambda subdir_node:
4614                       subdir_node.add_file(u"grandchild",
4615                                            upload.Data(DATA+"2",
4616                                                        convergence="")))
4617         d.addCallback(_stash_uri, "grandchild")
4618
4619         d.addCallback(lambda ign:
4620                       self.delete_shares_numbered(self.uris["subdir"],
4621                                                   range(1, 10)))
4622
4623         # root
4624         # root/good
4625         # root/small
4626         # root/sick
4627         # root/future
4628         # root/subdir [unrecoverable]
4629         # root/subdir/grandchild
4630
4631         # how should a streaming-JSON API indicate fatal error?
4632         # answer: emit ERROR: instead of a JSON string
4633
4634         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4635         def _check_broken_manifest(res):
4636             lines = res.splitlines()
4637             error_lines = [i
4638                            for (i,line) in enumerate(lines)
4639                            if line.startswith("ERROR:")]
4640             if not error_lines:
4641                 self.fail("no ERROR: in output: %s" % (res,))
4642             first_error = error_lines[0]
4643             error_line = lines[first_error]
4644             error_msg = lines[first_error+1:]
4645             error_msg_s = "\n".join(error_msg) + "\n"
4646             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4647                               error_line)
4648             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4649             units = [simplejson.loads(line) for line in lines[:first_error]]
4650             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4651             last_unit = units[-1]
4652             self.failUnlessEqual(last_unit["path"], ["subdir"])
4653         d.addCallback(_check_broken_manifest)
4654
4655         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4656         def _check_broken_deepcheck(res):
4657             lines = res.splitlines()
4658             error_lines = [i
4659                            for (i,line) in enumerate(lines)
4660                            if line.startswith("ERROR:")]
4661             if not error_lines:
4662                 self.fail("no ERROR: in output: %s" % (res,))
4663             first_error = error_lines[0]
4664             error_line = lines[first_error]
4665             error_msg = lines[first_error+1:]
4666             error_msg_s = "\n".join(error_msg) + "\n"
4667             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4668                               error_line)
4669             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4670             units = [simplejson.loads(line) for line in lines[:first_error]]
4671             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4672             last_unit = units[-1]
4673             self.failUnlessEqual(last_unit["path"], ["subdir"])
4674             r = last_unit["check-results"]["results"]
4675             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4676             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4677             self.failUnlessReallyEqual(r["recoverable"], False)
4678         d.addCallback(_check_broken_deepcheck)
4679
4680         d.addErrback(self.explain_web_error)
4681         return d
4682
4683     def test_deep_check_and_repair(self):
4684         self.basedir = "web/Grid/deep_check_and_repair"
4685         self.set_up_grid()
4686         c0 = self.g.clients[0]
4687         self.uris = {}
4688         self.fileurls = {}
4689         DATA = "data" * 100
4690         d = c0.create_dirnode()
4691         def _stash_root_and_create_file(n):
4692             self.rootnode = n
4693             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4694             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4695         d.addCallback(_stash_root_and_create_file)
4696         def _stash_uri(fn, which):
4697             self.uris[which] = fn.get_uri()
4698         d.addCallback(_stash_uri, "good")
4699         d.addCallback(lambda ign:
4700                       self.rootnode.add_file(u"small",
4701                                              upload.Data("literal",
4702                                                         convergence="")))
4703         d.addCallback(_stash_uri, "small")
4704         d.addCallback(lambda ign:
4705                       self.rootnode.add_file(u"sick",
4706                                              upload.Data(DATA+"1",
4707                                                         convergence="")))
4708         d.addCallback(_stash_uri, "sick")
4709         #d.addCallback(lambda ign:
4710         #              self.rootnode.add_file(u"dead",
4711         #                                     upload.Data(DATA+"2",
4712         #                                                convergence="")))
4713         #d.addCallback(_stash_uri, "dead")
4714
4715         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4716         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4717         #d.addCallback(_stash_uri, "corrupt")
4718
4719         def _clobber_shares(ignored):
4720             good_shares = self.find_uri_shares(self.uris["good"])
4721             self.failUnlessReallyEqual(len(good_shares), 10)
4722             sick_shares = self.find_uri_shares(self.uris["sick"])
4723             os.unlink(sick_shares[0][2])
4724             #dead_shares = self.find_uri_shares(self.uris["dead"])
4725             #for i in range(1, 10):
4726             #    os.unlink(dead_shares[i][2])
4727
4728             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4729             #cso = CorruptShareOptions()
4730             #cso.stdout = StringIO()
4731             #cso.parseOptions([c_shares[0][2]])
4732             #corrupt_share(cso)
4733         d.addCallback(_clobber_shares)
4734
4735         # root
4736         # root/good   CHK, 10 shares
4737         # root/small  LIT
4738         # root/sick   CHK, 9 shares
4739
4740         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4741         def _done(res):
4742             units = [simplejson.loads(line)
4743                      for line in res.splitlines()
4744                      if line]
4745             self.failUnlessReallyEqual(len(units), 4+1)
4746             # should be parent-first
4747             u0 = units[0]
4748             self.failUnlessEqual(u0["path"], [])
4749             self.failUnlessEqual(u0["type"], "directory")
4750             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4751             u0crr = u0["check-and-repair-results"]
4752             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4753             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4754
4755             ugood = [u for u in units
4756                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4757             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4758             ugoodcrr = ugood["check-and-repair-results"]
4759             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4760             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4761
4762             usick = [u for u in units
4763                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4764             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4765             usickcrr = usick["check-and-repair-results"]
4766             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4767             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4768             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4769             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4770
4771             stats = units[-1]
4772             self.failUnlessEqual(stats["type"], "stats")
4773             s = stats["stats"]
4774             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4775             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4776             self.failUnlessReallyEqual(s["count-directories"], 1)
4777         d.addCallback(_done)
4778
4779         d.addErrback(self.explain_web_error)
4780         return d
4781
4782     def _count_leases(self, ignored, which):
4783         u = self.uris[which]
4784         shares = self.find_uri_shares(u)
4785         lease_counts = []
4786         for shnum, serverid, fn in shares:
4787             sf = get_share_file(fn)
4788             num_leases = len(list(sf.get_leases()))
4789             lease_counts.append( (fn, num_leases) )
4790         return lease_counts
4791
4792     def _assert_leasecount(self, lease_counts, expected):
4793         for (fn, num_leases) in lease_counts:
4794             if num_leases != expected:
4795                 self.fail("expected %d leases, have %d, on %s" %
4796                           (expected, num_leases, fn))
4797
4798     def test_add_lease(self):
4799         self.basedir = "web/Grid/add_lease"
4800         self.set_up_grid(num_clients=2)
4801         c0 = self.g.clients[0]
4802         self.uris = {}
4803         DATA = "data" * 100
4804         d = c0.upload(upload.Data(DATA, convergence=""))
4805         def _stash_uri(ur, which):
4806             self.uris[which] = ur.uri
4807         d.addCallback(_stash_uri, "one")
4808         d.addCallback(lambda ign:
4809                       c0.upload(upload.Data(DATA+"1", convergence="")))
4810         d.addCallback(_stash_uri, "two")
4811         def _stash_mutable_uri(n, which):
4812             self.uris[which] = n.get_uri()
4813             assert isinstance(self.uris[which], str)
4814         d.addCallback(lambda ign:
4815             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4816         d.addCallback(_stash_mutable_uri, "mutable")
4817
4818         def _compute_fileurls(ignored):
4819             self.fileurls = {}
4820             for which in self.uris:
4821                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4822         d.addCallback(_compute_fileurls)
4823
4824         d.addCallback(self._count_leases, "one")
4825         d.addCallback(self._assert_leasecount, 1)
4826         d.addCallback(self._count_leases, "two")
4827         d.addCallback(self._assert_leasecount, 1)
4828         d.addCallback(self._count_leases, "mutable")
4829         d.addCallback(self._assert_leasecount, 1)
4830
4831         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4832         def _got_html_good(res):
4833             self.failUnless("Healthy" in res, res)
4834             self.failIf("Not Healthy" in res, res)
4835         d.addCallback(_got_html_good)
4836
4837         d.addCallback(self._count_leases, "one")
4838         d.addCallback(self._assert_leasecount, 1)
4839         d.addCallback(self._count_leases, "two")
4840         d.addCallback(self._assert_leasecount, 1)
4841         d.addCallback(self._count_leases, "mutable")
4842         d.addCallback(self._assert_leasecount, 1)
4843
4844         # this CHECK uses the original client, which uses the same
4845         # lease-secrets, so it will just renew the original lease
4846         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4847         d.addCallback(_got_html_good)
4848
4849         d.addCallback(self._count_leases, "one")
4850         d.addCallback(self._assert_leasecount, 1)
4851         d.addCallback(self._count_leases, "two")
4852         d.addCallback(self._assert_leasecount, 1)
4853         d.addCallback(self._count_leases, "mutable")
4854         d.addCallback(self._assert_leasecount, 1)
4855
4856         # this CHECK uses an alternate client, which adds a second lease
4857         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4858         d.addCallback(_got_html_good)
4859
4860         d.addCallback(self._count_leases, "one")
4861         d.addCallback(self._assert_leasecount, 2)
4862         d.addCallback(self._count_leases, "two")
4863         d.addCallback(self._assert_leasecount, 1)
4864         d.addCallback(self._count_leases, "mutable")
4865         d.addCallback(self._assert_leasecount, 1)
4866
4867         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4868         d.addCallback(_got_html_good)
4869
4870         d.addCallback(self._count_leases, "one")
4871         d.addCallback(self._assert_leasecount, 2)
4872         d.addCallback(self._count_leases, "two")
4873         d.addCallback(self._assert_leasecount, 1)
4874         d.addCallback(self._count_leases, "mutable")
4875         d.addCallback(self._assert_leasecount, 1)
4876
4877         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4878                       clientnum=1)
4879         d.addCallback(_got_html_good)
4880
4881         d.addCallback(self._count_leases, "one")
4882         d.addCallback(self._assert_leasecount, 2)
4883         d.addCallback(self._count_leases, "two")
4884         d.addCallback(self._assert_leasecount, 1)
4885         d.addCallback(self._count_leases, "mutable")
4886         d.addCallback(self._assert_leasecount, 2)
4887
4888         d.addErrback(self.explain_web_error)
4889         return d
4890
4891     def test_deep_add_lease(self):
4892         self.basedir = "web/Grid/deep_add_lease"
4893         self.set_up_grid(num_clients=2)
4894         c0 = self.g.clients[0]
4895         self.uris = {}
4896         self.fileurls = {}
4897         DATA = "data" * 100
4898         d = c0.create_dirnode()
4899         def _stash_root_and_create_file(n):
4900             self.rootnode = n
4901             self.uris["root"] = n.get_uri()
4902             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4903             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4904         d.addCallback(_stash_root_and_create_file)
4905         def _stash_uri(fn, which):
4906             self.uris[which] = fn.get_uri()
4907         d.addCallback(_stash_uri, "one")
4908         d.addCallback(lambda ign:
4909                       self.rootnode.add_file(u"small",
4910                                              upload.Data("literal",
4911                                                         convergence="")))
4912         d.addCallback(_stash_uri, "small")
4913
4914         d.addCallback(lambda ign:
4915             c0.create_mutable_file(publish.MutableData("mutable")))
4916         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4917         d.addCallback(_stash_uri, "mutable")
4918
4919         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4920         def _done(res):
4921             units = [simplejson.loads(line)
4922                      for line in res.splitlines()
4923                      if line]
4924             # root, one, small, mutable,   stats
4925             self.failUnlessReallyEqual(len(units), 4+1)
4926         d.addCallback(_done)
4927
4928         d.addCallback(self._count_leases, "root")
4929         d.addCallback(self._assert_leasecount, 1)
4930         d.addCallback(self._count_leases, "one")
4931         d.addCallback(self._assert_leasecount, 1)
4932         d.addCallback(self._count_leases, "mutable")
4933         d.addCallback(self._assert_leasecount, 1)
4934
4935         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4936         d.addCallback(_done)
4937
4938         d.addCallback(self._count_leases, "root")
4939         d.addCallback(self._assert_leasecount, 1)
4940         d.addCallback(self._count_leases, "one")
4941         d.addCallback(self._assert_leasecount, 1)
4942         d.addCallback(self._count_leases, "mutable")
4943         d.addCallback(self._assert_leasecount, 1)
4944
4945         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4946                       clientnum=1)
4947         d.addCallback(_done)
4948
4949         d.addCallback(self._count_leases, "root")
4950         d.addCallback(self._assert_leasecount, 2)
4951         d.addCallback(self._count_leases, "one")
4952         d.addCallback(self._assert_leasecount, 2)
4953         d.addCallback(self._count_leases, "mutable")
4954         d.addCallback(self._assert_leasecount, 2)
4955
4956         d.addErrback(self.explain_web_error)
4957         return d
4958
4959
4960     def test_exceptions(self):
4961         self.basedir = "web/Grid/exceptions"
4962         self.set_up_grid(num_clients=1, num_servers=2)
4963         c0 = self.g.clients[0]
4964         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4965         self.fileurls = {}
4966         DATA = "data" * 100
4967         d = c0.create_dirnode()
4968         def _stash_root(n):
4969             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4970             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4971             return n
4972         d.addCallback(_stash_root)
4973         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4974         def _stash_bad(ur):
4975             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4976             self.delete_shares_numbered(ur.uri, range(1,10))
4977
4978             u = uri.from_string(ur.uri)
4979             u.key = testutil.flip_bit(u.key, 0)
4980             baduri = u.to_string()
4981             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4982         d.addCallback(_stash_bad)
4983         d.addCallback(lambda ign: c0.create_dirnode())
4984         def _mangle_dirnode_1share(n):
4985             u = n.get_uri()
4986             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4987             self.fileurls["dir-1share-json"] = url + "?t=json"
4988             self.delete_shares_numbered(u, range(1,10))
4989         d.addCallback(_mangle_dirnode_1share)
4990         d.addCallback(lambda ign: c0.create_dirnode())
4991         def _mangle_dirnode_0share(n):
4992             u = n.get_uri()
4993             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4994             self.fileurls["dir-0share-json"] = url + "?t=json"
4995             self.delete_shares_numbered(u, range(0,10))
4996         d.addCallback(_mangle_dirnode_0share)
4997
4998         # NotEnoughSharesError should be reported sensibly, with a
4999         # text/plain explanation of the problem, and perhaps some
5000         # information on which shares *could* be found.
5001
5002         d.addCallback(lambda ignored:
5003                       self.shouldHTTPError("GET unrecoverable",
5004                                            410, "Gone", "NoSharesError",
5005                                            self.GET, self.fileurls["0shares"]))
5006         def _check_zero_shares(body):
5007             self.failIf("<html>" in body, body)
5008             body = " ".join(body.strip().split())
5009             exp = ("NoSharesError: no shares could be found. "
5010                    "Zero shares usually indicates a corrupt URI, or that "
5011                    "no servers were connected, but it might also indicate "
5012                    "severe corruption. You should perform a filecheck on "
5013                    "this object to learn more. The full error message is: "
5014                    "no shares (need 3). Last failure: None")
5015             self.failUnlessReallyEqual(exp, body)
5016         d.addCallback(_check_zero_shares)
5017
5018
5019         d.addCallback(lambda ignored:
5020                       self.shouldHTTPError("GET 1share",
5021                                            410, "Gone", "NotEnoughSharesError",
5022                                            self.GET, self.fileurls["1share"]))
5023         def _check_one_share(body):
5024             self.failIf("<html>" in body, body)
5025             body = " ".join(body.strip().split())
5026             msgbase = ("NotEnoughSharesError: This indicates that some "
5027                        "servers were unavailable, or that shares have been "
5028                        "lost to server departure, hard drive failure, or disk "
5029                        "corruption. You should perform a filecheck on "
5030                        "this object to learn more. The full error message is:"
5031                        )
5032             msg1 = msgbase + (" ran out of shares:"
5033                               " complete=sh0"
5034                               " pending="
5035                               " overdue= unused= need 3. Last failure: None")
5036             msg2 = msgbase + (" ran out of shares:"
5037                               " complete="
5038                               " pending=Share(sh0-on-xgru5)"
5039                               " overdue= unused= need 3. Last failure: None")
5040             self.failUnless(body == msg1 or body == msg2, body)
5041         d.addCallback(_check_one_share)
5042
5043         d.addCallback(lambda ignored:
5044                       self.shouldHTTPError("GET imaginary",
5045                                            404, "Not Found", None,
5046                                            self.GET, self.fileurls["imaginary"]))
5047         def _missing_child(body):
5048             self.failUnless("No such child: imaginary" in body, body)
5049         d.addCallback(_missing_child)
5050
5051         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5052         def _check_0shares_dir_html(body):
5053             self.failUnless("<html>" in body, body)
5054             # we should see the regular page, but without the child table or
5055             # the dirops forms
5056             body = " ".join(body.strip().split())
5057             self.failUnlessIn('href="?t=info">More info on this directory',
5058                               body)
5059             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5060                    "could not be retrieved, because there were insufficient "
5061                    "good shares. This might indicate that no servers were "
5062                    "connected, insufficient servers were connected, the URI "
5063                    "was corrupt, or that shares have been lost due to server "
5064                    "departure, hard drive failure, or disk corruption. You "
5065                    "should perform a filecheck on this object to learn more.")
5066             self.failUnlessIn(exp, body)
5067             self.failUnlessIn("No upload forms: directory is unreadable", body)
5068         d.addCallback(_check_0shares_dir_html)
5069
5070         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5071         def _check_1shares_dir_html(body):
5072             # at some point, we'll split UnrecoverableFileError into 0-shares
5073             # and some-shares like we did for immutable files (since there
5074             # are different sorts of advice to offer in each case). For now,
5075             # they present the same way.
5076             self.failUnless("<html>" in body, body)
5077             body = " ".join(body.strip().split())
5078             self.failUnlessIn('href="?t=info">More info on this directory',
5079                               body)
5080             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5081                    "could not be retrieved, because there were insufficient "
5082                    "good shares. This might indicate that no servers were "
5083                    "connected, insufficient servers were connected, the URI "
5084                    "was corrupt, or that shares have been lost due to server "
5085                    "departure, hard drive failure, or disk corruption. You "
5086                    "should perform a filecheck on this object to learn more.")
5087             self.failUnlessIn(exp, body)
5088             self.failUnlessIn("No upload forms: directory is unreadable", body)
5089         d.addCallback(_check_1shares_dir_html)
5090
5091         d.addCallback(lambda ignored:
5092                       self.shouldHTTPError("GET dir-0share-json",
5093                                            410, "Gone", "UnrecoverableFileError",
5094                                            self.GET,
5095                                            self.fileurls["dir-0share-json"]))
5096         def _check_unrecoverable_file(body):
5097             self.failIf("<html>" in body, body)
5098             body = " ".join(body.strip().split())
5099             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5100                    "could not be retrieved, because there were insufficient "
5101                    "good shares. This might indicate that no servers were "
5102                    "connected, insufficient servers were connected, the URI "
5103                    "was corrupt, or that shares have been lost due to server "
5104                    "departure, hard drive failure, or disk corruption. You "
5105                    "should perform a filecheck on this object to learn more.")
5106             self.failUnlessReallyEqual(exp, body)
5107         d.addCallback(_check_unrecoverable_file)
5108
5109         d.addCallback(lambda ignored:
5110                       self.shouldHTTPError("GET dir-1share-json",
5111                                            410, "Gone", "UnrecoverableFileError",
5112                                            self.GET,
5113                                            self.fileurls["dir-1share-json"]))
5114         d.addCallback(_check_unrecoverable_file)
5115
5116         d.addCallback(lambda ignored:
5117                       self.shouldHTTPError("GET imaginary",
5118                                            404, "Not Found", None,
5119                                            self.GET, self.fileurls["imaginary"]))
5120
5121         # attach a webapi child that throws a random error, to test how it
5122         # gets rendered.
5123         w = c0.getServiceNamed("webish")
5124         w.root.putChild("ERRORBOOM", ErrorBoom())
5125
5126         # "Accept: */*" :        should get a text/html stack trace
5127         # "Accept: text/plain" : should get a text/plain stack trace
5128         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5129         # no Accept header:      should get a text/html stack trace
5130
5131         d.addCallback(lambda ignored:
5132                       self.shouldHTTPError("GET errorboom_html",
5133                                            500, "Internal Server Error", None,
5134                                            self.GET, "ERRORBOOM",
5135                                            headers={"accept": ["*/*"]}))
5136         def _internal_error_html1(body):
5137             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5138         d.addCallback(_internal_error_html1)
5139
5140         d.addCallback(lambda ignored:
5141                       self.shouldHTTPError("GET errorboom_text",
5142                                            500, "Internal Server Error", None,
5143                                            self.GET, "ERRORBOOM",
5144                                            headers={"accept": ["text/plain"]}))
5145         def _internal_error_text2(body):
5146             self.failIf("<html>" in body, body)
5147             self.failUnless(body.startswith("Traceback "), body)
5148         d.addCallback(_internal_error_text2)
5149
5150         CLI_accepts = "text/plain, application/octet-stream"
5151         d.addCallback(lambda ignored:
5152                       self.shouldHTTPError("GET errorboom_text",
5153                                            500, "Internal Server Error", None,
5154                                            self.GET, "ERRORBOOM",
5155                                            headers={"accept": [CLI_accepts]}))
5156         def _internal_error_text3(body):
5157             self.failIf("<html>" in body, body)
5158             self.failUnless(body.startswith("Traceback "), body)
5159         d.addCallback(_internal_error_text3)
5160
5161         d.addCallback(lambda ignored:
5162                       self.shouldHTTPError("GET errorboom_text",
5163                                            500, "Internal Server Error", None,
5164                                            self.GET, "ERRORBOOM"))
5165         def _internal_error_html4(body):
5166             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5167         d.addCallback(_internal_error_html4)
5168
5169         def _flush_errors(res):
5170             # Trial: please ignore the CompletelyUnhandledError in the logs
5171             self.flushLoggedErrors(CompletelyUnhandledError)
5172             return res
5173         d.addBoth(_flush_errors)
5174
5175         return d
5176
5177     def test_blacklist(self):
5178         # download from a blacklisted URI, get an error
5179         self.basedir = "web/Grid/blacklist"
5180         self.set_up_grid()
5181         c0 = self.g.clients[0]
5182         c0_basedir = c0.basedir
5183         fn = os.path.join(c0_basedir, "access.blacklist")
5184         self.uris = {}
5185         DATA = "off-limits " * 50
5186
5187         d = c0.upload(upload.Data(DATA, convergence=""))
5188         def _stash_uri_and_create_dir(ur):
5189             self.uri = ur.uri
5190             self.url = "uri/"+self.uri
5191             u = uri.from_string_filenode(self.uri)
5192             self.si = u.get_storage_index()
5193             childnode = c0.create_node_from_uri(self.uri, None)
5194             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5195         d.addCallback(_stash_uri_and_create_dir)
5196         def _stash_dir(node):
5197             self.dir_node = node
5198             self.dir_uri = node.get_uri()
5199             self.dir_url = "uri/"+self.dir_uri
5200         d.addCallback(_stash_dir)
5201         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5202         def _check_dir_html(body):
5203             self.failUnlessIn("<html>", body)
5204             self.failUnlessIn("blacklisted.txt</a>", body)
5205         d.addCallback(_check_dir_html)
5206         d.addCallback(lambda ign: self.GET(self.url))
5207         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5208
5209         def _blacklist(ign):
5210             f = open(fn, "w")
5211             f.write(" # this is a comment\n")
5212             f.write(" \n")
5213             f.write("\n") # also exercise blank lines
5214             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5215             f.close()
5216             # clients should be checking the blacklist each time, so we don't
5217             # need to restart the client
5218         d.addCallback(_blacklist)
5219         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5220                                                        403, "Forbidden",
5221                                                        "Access Prohibited: off-limits",
5222                                                        self.GET, self.url))
5223
5224         # We should still be able to list the parent directory, in HTML...
5225         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5226         def _check_dir_html2(body):
5227             self.failUnlessIn("<html>", body)
5228             self.failUnlessIn("blacklisted.txt</strike>", body)
5229         d.addCallback(_check_dir_html2)
5230
5231         # ... and in JSON (used by CLI).
5232         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5233         def _check_dir_json(res):
5234             data = simplejson.loads(res)
5235             self.failUnless(isinstance(data, list), data)
5236             self.failUnlessEqual(data[0], "dirnode")
5237             self.failUnless(isinstance(data[1], dict), data)
5238             self.failUnlessIn("children", data[1])
5239             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5240             childdata = data[1]["children"]["blacklisted.txt"]
5241             self.failUnless(isinstance(childdata, list), data)
5242             self.failUnlessEqual(childdata[0], "filenode")
5243             self.failUnless(isinstance(childdata[1], dict), data)
5244         d.addCallback(_check_dir_json)
5245
5246         def _unblacklist(ign):
5247             open(fn, "w").close()
5248             # the Blacklist object watches mtime to tell when the file has
5249             # changed, but on windows this test will run faster than the
5250             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5251             # to force a reload.
5252             self.g.clients[0].blacklist.last_mtime -= 2.0
5253         d.addCallback(_unblacklist)
5254
5255         # now a read should work
5256         d.addCallback(lambda ign: self.GET(self.url))
5257         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5258
5259         # read again to exercise the blacklist-is-unchanged logic
5260         d.addCallback(lambda ign: self.GET(self.url))
5261         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5262
5263         # now add a blacklisted directory, and make sure files under it are
5264         # refused too
5265         def _add_dir(ign):
5266             childnode = c0.create_node_from_uri(self.uri, None)
5267             return c0.create_dirnode({u"child": (childnode,{}) })
5268         d.addCallback(_add_dir)
5269         def _get_dircap(dn):
5270             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5271             self.dir_url_base = "uri/"+dn.get_write_uri()
5272             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5273             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5274             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5275             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5276         d.addCallback(_get_dircap)
5277         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5278         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5279         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5280         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5281         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5282         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5283         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5284         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5285         d.addCallback(lambda ign: self.GET(self.child_url))
5286         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5287
5288         def _block_dir(ign):
5289             f = open(fn, "w")
5290             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5291             f.close()
5292             self.g.clients[0].blacklist.last_mtime -= 2.0
5293         d.addCallback(_block_dir)
5294         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5295                                                        403, "Forbidden",
5296                                                        "Access Prohibited: dir-off-limits",
5297                                                        self.GET, self.dir_url_base))
5298         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5299                                                        403, "Forbidden",
5300                                                        "Access Prohibited: dir-off-limits",
5301                                                        self.GET, self.dir_url_json1))
5302         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5303                                                        403, "Forbidden",
5304                                                        "Access Prohibited: dir-off-limits",
5305                                                        self.GET, self.dir_url_json2))
5306         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5307                                                        403, "Forbidden",
5308                                                        "Access Prohibited: dir-off-limits",
5309                                                        self.GET, self.dir_url_json_ro))
5310         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5311                                                        403, "Forbidden",
5312                                                        "Access Prohibited: dir-off-limits",
5313                                                        self.GET, self.child_url))
5314         return d
5315
5316
5317 class CompletelyUnhandledError(Exception):
5318     pass
5319 class ErrorBoom(rend.Page):
5320     def beforeRender(self, ctx):
5321         raise CompletelyUnhandledError("whoops")