src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649         d.addCallback(_check_dl_json)
 650         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 651         def _check_ul(res):
 652             self.failUnless("File Upload Status" in res, res)
 653         d.addCallback(_check_ul)
 654         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 655         def _check_mapupdate(res):
 656             self.failUnless("Mutable File Servermap Update Status" in res, res)
 657         d.addCallback(_check_mapupdate)
 658         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 659         def _check_publish(res):
 660             self.failUnless("Mutable File Publish Status" in res, res)
 661         d.addCallback(_check_publish)
 662         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 663         def _check_retrieve(res):
 664             self.failUnless("Mutable File Retrieve Status" in res, res)
 665         d.addCallback(_check_retrieve)
 666
 667         return d
 668
 669     def test_status_numbers(self):
 670         drrm = status.DownloadResultsRendererMixin()
 671         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 680
 681         urrm = status.UploadResultsRendererMixin()
 682         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 691
 692     def test_GET_FILEURL(self):
 693         d = self.GET(self.public_url + "/foo/bar.txt")
 694         d.addCallback(self.failUnlessIsBarDotTxt)
 695         return d
 696
 697     def test_GET_FILEURL_range(self):
 698         headers = {"range": "bytes=1-10"}
 699         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 700                      return_response=True)
 701         def _got((res, status, headers)):
 702             self.failUnlessReallyEqual(int(status), 206)
 703             self.failUnless(headers.has_key("content-range"))
 704             self.failUnlessReallyEqual(headers["content-range"][0],
 705                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 706             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 707         d.addCallback(_got)
 708         return d
 709
 710     def test_GET_FILEURL_partial_range(self):
 711         headers = {"range": "bytes=5-"}
 712         length  = len(self.BAR_CONTENTS)
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 206)
 717             self.failUnless(headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(headers["content-range"][0],
 719                                        "bytes 5-%d/%d" % (length-1, length))
 720             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 721         d.addCallback(_got)
 722         return d
 723
 724     def test_GET_FILEURL_partial_end_range(self):
 725         headers = {"range": "bytes=-5"}
 726         length  = len(self.BAR_CONTENTS)
 727         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 728                      return_response=True)
 729         def _got((res, status, headers)):
 730             self.failUnlessReallyEqual(int(status), 206)
 731             self.failUnless(headers.has_key("content-range"))
 732             self.failUnlessReallyEqual(headers["content-range"][0],
 733                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 734             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_partial_range_overrun(self):
 739         headers = {"range": "bytes=100-200"}
 740         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 741                              "416 Requested Range not satisfiable",
 742                              "First beyond end of file",
 743                              self.GET, self.public_url + "/foo/bar.txt",
 744                              headers=headers)
 745         return d
 746
 747     def test_HEAD_FILEURL_range(self):
 748         headers = {"range": "bytes=1-10"}
 749         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 750                      return_response=True)
 751         def _got((res, status, headers)):
 752             self.failUnlessReallyEqual(res, "")
 753             self.failUnlessReallyEqual(int(status), 206)
 754             self.failUnless(headers.has_key("content-range"))
 755             self.failUnlessReallyEqual(headers["content-range"][0],
 756                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_HEAD_FILEURL_partial_range(self):
 761         headers = {"range": "bytes=5-"}
 762         length  = len(self.BAR_CONTENTS)
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 764                      return_response=True)
 765         def _got((res, status, headers)):
 766             self.failUnlessReallyEqual(int(status), 206)
 767             self.failUnless(headers.has_key("content-range"))
 768             self.failUnlessReallyEqual(headers["content-range"][0],
 769                                        "bytes 5-%d/%d" % (length-1, length))
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_HEAD_FILEURL_partial_end_range(self):
 774         headers = {"range": "bytes=-5"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range_overrun(self):
 787         headers = {"range": "bytes=100-200"}
 788         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 789                              "416 Requested Range not satisfiable",
 790                              "",
 791                              self.HEAD, self.public_url + "/foo/bar.txt",
 792                              headers=headers)
 793         return d
 794
 795     def test_GET_FILEURL_range_bad(self):
 796         headers = {"range": "BOGUS=fizbop-quarnak"}
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 200)
 801             self.failUnless(not headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 803         d.addCallback(_got)
 804         return d
 805
 806     def test_HEAD_FILEURL(self):
 807         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 808         def _got((res, status, headers)):
 809             self.failUnlessReallyEqual(res, "")
 810             self.failUnlessReallyEqual(headers["content-length"][0],
 811                                        str(len(self.BAR_CONTENTS)))
 812             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 813         d.addCallback(_got)
 814         return d
 815
 816     def test_GET_FILEURL_named(self):
 817         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 818         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 819         d = self.GET(base + "/@@name=/blah.txt")
 820         d.addCallback(self.failUnlessIsBarDotTxt)
 821         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 822         d.addCallback(self.failUnlessIsBarDotTxt)
 823         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 824         d.addCallback(self.failUnlessIsBarDotTxt)
 825         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 826         d.addCallback(self.failUnlessIsBarDotTxt)
 827         save_url = base + "?save=true&filename=blah.txt"
 828         d.addCallback(lambda res: self.GET(save_url))
 829         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 830         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 831         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 832         u_url = base + "?save=true&filename=" + u_fn_e
 833         d.addCallback(lambda res: self.GET(u_url))
 834         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 835         return d
 836
 837     def test_PUT_FILEURL_named_bad(self):
 838         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 839         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 840                              "400 Bad Request",
 841                              "/file can only be used with GET or HEAD",
 842                              self.PUT, base + "/@@name=/blah.txt", "")
 843         return d
 844
 845
 846     def test_GET_DIRURL_named_bad(self):
 847         base = "/file/%s" % urllib.quote(self._foo_uri)
 848         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 849                              "400 Bad Request",
 850                              "is not a file-cap",
 851                              self.GET, base + "/@@name=/blah.txt")
 852         return d
 853
 854     def test_GET_slash_file_bad(self):
 855         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 856                              "404 Not Found",
 857                              "/file must be followed by a file-cap and a name",
 858                              self.GET, "/file")
 859         return d
 860
 861     def test_GET_unhandled_URI_named(self):
 862         contents, n, newuri = self.makefile(12)
 863         verifier_cap = n.get_verify_cap().to_string()
 864         base = "/file/%s" % urllib.quote(verifier_cap)
 865         # client.create_node_from_uri() can't handle verify-caps
 866         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 867                              "400 Bad Request", "is not a file-cap",
 868                              self.GET, base)
 869         return d
 870
 871     def test_GET_unhandled_URI(self):
 872         contents, n, newuri = self.makefile(12)
 873         verifier_cap = n.get_verify_cap().to_string()
 874         base = "/uri/%s" % urllib.quote(verifier_cap)
 875         # client.create_node_from_uri() can't handle verify-caps
 876         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 877                              "400 Bad Request",
 878                              "GET unknown URI type: can only do t=info",
 879                              self.GET, base)
 880         return d
 881
 882     def test_GET_FILE_URI(self):
 883         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 884         d = self.GET(base)
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         return d
 887
 888     def test_GET_FILE_URI_mdmf(self):
 889         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 890         d = self.GET(base)
 891         d.addCallback(self.failUnlessIsQuuxDotTxt)
 892         return d
 893
 894     def test_GET_FILE_URI_mdmf_extensions(self):
 895         base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
 896         d = self.GET(base)
 897         d.addCallback(self.failUnlessIsQuuxDotTxt)
 898         return d
 899
 900     def test_GET_FILE_URI_mdmf_bare_cap(self):
 901         cap_elements = self._quux_txt_uri.split(":")
 902         # 6 == expected cap length with two extensions.
 903         self.failUnlessEqual(len(cap_elements), 6)
 904
 905         # Now lop off the extension parameters and stitch everything
 906         # back together
 907         quux_uri = ":".join(cap_elements[:len(cap_elements) - 2])
 908
 909         # Now GET that. We should get back quux.
 910         base = "/uri/%s" % urllib.quote(quux_uri)
 911         d = self.GET(base)
 912         d.addCallback(self.failUnlessIsQuuxDotTxt)
 913         return d
 914
 915     def test_GET_FILE_URI_mdmf_readonly(self):
 916         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 917         d = self.GET(base)
 918         d.addCallback(self.failUnlessIsQuuxDotTxt)
 919         return d
 920
 921     def test_GET_FILE_URI_badchild(self):
 922         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 923         errmsg = "Files have no children, certainly not named 'boguschild'"
 924         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 925                              "400 Bad Request", errmsg,
 926                              self.GET, base)
 927         return d
 928
 929     def test_PUT_FILE_URI_badchild(self):
 930         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 931         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 932         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 933                              "400 Bad Request", errmsg,
 934                              self.PUT, base, "")
 935         return d
 936
 937     def test_PUT_FILE_URI_mdmf(self):
 938         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 939         self._quux_new_contents = "new_contents"
 940         d = self.GET(base)
 941         d.addCallback(lambda res:
 942             self.failUnlessIsQuuxDotTxt(res))
 943         d.addCallback(lambda ignored:
 944             self.PUT(base, self._quux_new_contents))
 945         d.addCallback(lambda ignored:
 946             self.GET(base))
 947         d.addCallback(lambda res:
 948             self.failUnlessReallyEqual(res, self._quux_new_contents))
 949         return d
 950
 951     def test_PUT_FILE_URI_mdmf_extensions(self):
 952         base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
 953         self._quux_new_contents = "new_contents"
 954         d = self.GET(base)
 955         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 956         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 957         d.addCallback(lambda ignored: self.GET(base))
 958         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 959                                                        res))
 960         return d
 961
 962     def test_PUT_FILE_URI_mdmf_bare_cap(self):
 963         elements = self._quux_txt_uri.split(":")
 964         self.failUnlessEqual(len(elements), 6)
 965
 966         quux_uri = ":".join(elements[:len(elements) - 2])
 967         base = "/uri/%s" % urllib.quote(quux_uri)
 968         self._quux_new_contents = "new_contents" * 50000
 969
 970         d = self.GET(base)
 971         d.addCallback(self.failUnlessIsQuuxDotTxt)
 972         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 973         d.addCallback(lambda ignored: self.GET(base))
 974         d.addCallback(lambda res:
 975             self.failUnlessEqual(res, self._quux_new_contents))
 976         return d
 977
 978     def test_PUT_FILE_URI_mdmf_readonly(self):
 979         # We're not allowed to PUT things to a readonly cap.
 980         base = "/uri/%s" % self._quux_txt_readonly_uri
 981         d = self.GET(base)
 982         d.addCallback(lambda res:
 983             self.failUnlessIsQuuxDotTxt(res))
 984         # What should we get here? We get a 500 error now; that's not right.
 985         d.addCallback(lambda ignored:
 986             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 987                              "400 Bad Request", "read-only cap",
 988                              self.PUT, base, "new data"))
 989         return d
 990
 991     def test_PUT_FILE_URI_sdmf_readonly(self):
 992         # We're not allowed to put things to a readonly cap.
 993         base = "/uri/%s" % self._baz_txt_readonly_uri
 994         d = self.GET(base)
 995         d.addCallback(lambda res:
 996             self.failUnlessIsBazDotTxt(res))
 997         d.addCallback(lambda ignored:
 998             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 999                              "400 Bad Request", "read-only cap",
1000                              self.PUT, base, "new_data"))
1001         return d
1002
1003     # TODO: version of this with a Unicode filename
1004     def test_GET_FILEURL_save(self):
1005         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1006                      return_response=True)
1007         def _got((res, statuscode, headers)):
1008             content_disposition = headers["content-disposition"][0]
1009             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1010             self.failUnlessIsBarDotTxt(res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_missing(self):
1015         d = self.GET(self.public_url + "/foo/missing")
1016         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1017         return d
1018
1019     def test_GET_FILEURL_info_mdmf(self):
1020         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1021         def _got(res):
1022             self.failUnlessIn("mutable file (mdmf)", res)
1023             self.failUnlessIn(self._quux_txt_uri, res)
1024             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1025         d.addCallback(_got)
1026         return d
1027
1028     def test_GET_FILEURL_info_mdmf_readonly(self):
1029         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1030         def _got(res):
1031             self.failUnlessIn("mutable file (mdmf)", res)
1032             self.failIfIn(self._quux_txt_uri, res)
1033             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1034         d.addCallback(_got)
1035         return d
1036
1037     def test_GET_FILEURL_info_sdmf(self):
1038         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1039         def _got(res):
1040             self.failUnlessIn("mutable file (sdmf)", res)
1041             self.failUnlessIn(self._baz_txt_uri, res)
1042         d.addCallback(_got)
1043         return d
1044
1045     def test_GET_FILEURL_info_mdmf_extensions(self):
1046         d = self.GET("/uri/%s:3:131073?t=info" % self._quux_txt_uri)
1047         def _got(res):
1048             self.failUnlessIn("mutable file (mdmf)", res)
1049             self.failUnlessIn(self._quux_txt_uri, res)
1050             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1051         d.addCallback(_got)
1052         return d
1053
1054     def test_GET_FILEURL_info_mdmf_bare_cap(self):
1055         elements = self._quux_txt_uri.split(":")
1056         self.failUnlessEqual(len(elements), 6)
1057
1058         quux_uri = ":".join(elements[:len(elements) - 2])
1059         base = "/uri/%s?t=info" % urllib.quote(quux_uri)
1060         d = self.GET(base)
1061         def _got(res):
1062             self.failUnlessIn("mutable file (mdmf)", res)
1063             self.failUnlessIn(quux_uri, res)
1064         d.addCallback(_got)
1065         return d
1066
1067     def test_PUT_overwrite_only_files(self):
1068         # create a directory, put a file in that directory.
1069         contents, n, filecap = self.makefile(8)
1070         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1071         d.addCallback(lambda res:
1072             self.PUT(self.public_url + "/foo/dir/file1.txt",
1073                      self.NEWFILE_CONTENTS))
1074         # try to overwrite the file with replace=only-files
1075         # (this should work)
1076         d.addCallback(lambda res:
1077             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1078                      filecap))
1079         d.addCallback(lambda res:
1080             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1081                  "There was already a child by that name, and you asked me "
1082                  "to not replace it",
1083                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1084                  filecap))
1085         return d
1086
1087     def test_PUT_NEWFILEURL(self):
1088         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1089         # TODO: we lose the response code, so we can't check this
1090         #self.failUnlessReallyEqual(responsecode, 201)
1091         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1092         d.addCallback(lambda res:
1093                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1094                                                       self.NEWFILE_CONTENTS))
1095         return d
1096
1097     def test_PUT_NEWFILEURL_not_mutable(self):
1098         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1099                      self.NEWFILE_CONTENTS)
1100         # TODO: we lose the response code, so we can't check this
1101         #self.failUnlessReallyEqual(responsecode, 201)
1102         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1103         d.addCallback(lambda res:
1104                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1105                                                       self.NEWFILE_CONTENTS))
1106         return d
1107
1108     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1109         # this should get us a few segments of an MDMF mutable file,
1110         # which we can then test for.
1111         contents = self.NEWFILE_CONTENTS * 300000
1112         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1113                      contents)
1114         def _got_filecap(filecap):
1115             self.failUnless(filecap.startswith("URI:MDMF"))
1116             return filecap
1117         d.addCallback(_got_filecap)
1118         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1119         d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1120         return d
1121
1122     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1123         contents = self.NEWFILE_CONTENTS * 300000
1124         d = self.PUT("/uri?mutable=true&mutable-type=sdmf",
1125                      contents)
1126         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1127         d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1128         return d
1129
1130     def test_PUT_NEWFILEURL_unlinked_bad_mutable_type(self):
1131         contents = self.NEWFILE_CONTENTS * 300000
1132         return self.shouldHTTPError("test bad mutable type",
1133                                     400, "Bad Request", "Unknown type: foo",
1134                                     self.PUT, "/uri?mutable=true&mutable-type=foo",
1135                                     contents)
1136
1137     def test_PUT_NEWFILEURL_range_bad(self):
1138         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1139         target = self.public_url + "/foo/new.txt"
1140         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1141                              "501 Not Implemented",
1142                              "Content-Range in PUT not yet supported",
1143                              # (and certainly not for immutable files)
1144                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1145                              headers=headers)
1146         d.addCallback(lambda res:
1147                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1148         return d
1149
1150     def test_PUT_NEWFILEURL_mutable(self):
1151         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1152                      self.NEWFILE_CONTENTS)
1153         # TODO: we lose the response code, so we can't check this
1154         #self.failUnlessReallyEqual(responsecode, 201)
1155         def _check_uri(res):
1156             u = uri.from_string_mutable_filenode(res)
1157             self.failUnless(u.is_mutable())
1158             self.failIf(u.is_readonly())
1159             return res
1160         d.addCallback(_check_uri)
1161         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1162         d.addCallback(lambda res:
1163                       self.failUnlessMutableChildContentsAre(self._foo_node,
1164                                                              u"new.txt",
1165                                                              self.NEWFILE_CONTENTS))
1166         return d
1167
1168     def test_PUT_NEWFILEURL_mutable_toobig(self):
1169         # It is okay to upload large mutable files, so we should be able
1170         # to do that.
1171         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1172                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1173         return d
1174
1175     def test_PUT_NEWFILEURL_replace(self):
1176         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1177         # TODO: we lose the response code, so we can't check this
1178         #self.failUnlessReallyEqual(responsecode, 200)
1179         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1180         d.addCallback(lambda res:
1181                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1182                                                       self.NEWFILE_CONTENTS))
1183         return d
1184
1185     def test_PUT_NEWFILEURL_bad_t(self):
1186         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1187                              "PUT to a file: bad t=bogus",
1188                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1189                              "contents")
1190         return d
1191
1192     def test_PUT_NEWFILEURL_no_replace(self):
1193         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1194                      self.NEWFILE_CONTENTS)
1195         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1196                   "409 Conflict",
1197                   "There was already a child by that name, and you asked me "
1198                   "to not replace it")
1199         return d
1200
1201     def test_PUT_NEWFILEURL_mkdirs(self):
1202         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1203         fn = self._foo_node
1204         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1205         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1206         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1207         d.addCallback(lambda res:
1208                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1209                                                       self.NEWFILE_CONTENTS))
1210         return d
1211
1212     def test_PUT_NEWFILEURL_blocked(self):
1213         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1214                      self.NEWFILE_CONTENTS)
1215         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1216                   "409 Conflict",
1217                   "Unable to create directory 'blockingfile': a file was in the way")
1218         return d
1219
1220     def test_PUT_NEWFILEURL_emptyname(self):
1221         # an empty pathname component (i.e. a double-slash) is disallowed
1222         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1223                              "400 Bad Request",
1224                              "The webapi does not allow empty pathname components",
1225                              self.PUT, self.public_url + "/foo//new.txt", "")
1226         return d
1227
1228     def test_DELETE_FILEURL(self):
1229         d = self.DELETE(self.public_url + "/foo/bar.txt")
1230         d.addCallback(lambda res:
1231                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1232         return d
1233
1234     def test_DELETE_FILEURL_missing(self):
1235         d = self.DELETE(self.public_url + "/foo/missing")
1236         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1237         return d
1238
1239     def test_DELETE_FILEURL_missing2(self):
1240         d = self.DELETE(self.public_url + "/missing/missing")
1241         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1242         return d
1243
1244     def failUnlessHasBarDotTxtMetadata(self, res):
1245         data = simplejson.loads(res)
1246         self.failUnless(isinstance(data, list))
1247         self.failUnlessIn("metadata", data[1])
1248         self.failUnlessIn("tahoe", data[1]["metadata"])
1249         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1250         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1251         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1252                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1253
1254     def test_GET_FILEURL_json(self):
1255         # twisted.web.http.parse_qs ignores any query args without an '=', so
1256         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1257         # instead. This may make it tricky to emulate the S3 interface
1258         # completely.
1259         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1260         def _check1(data):
1261             self.failUnlessIsBarJSON(data)
1262             self.failUnlessHasBarDotTxtMetadata(data)
1263             return
1264         d.addCallback(_check1)
1265         return d
1266
1267     def test_GET_FILEURL_json_mutable_type(self):
1268         # The JSON should include mutable-type, which says whether the
1269         # file is SDMF or MDMF
1270         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1271                      self.NEWFILE_CONTENTS * 300000)
1272         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1273         def _got_json(json, version):
1274             data = simplejson.loads(json)
1275             assert "filenode" == data[0]
1276             data = data[1]
1277             assert isinstance(data, dict)
1278
1279             self.failUnlessIn("mutable-type", data)
1280             self.failUnlessEqual(data['mutable-type'], version)
1281
1282         d.addCallback(_got_json, "mdmf")
1283         # Now make an SDMF file and check that it is reported correctly.
1284         d.addCallback(lambda ignored:
1285             self.PUT("/uri?mutable=true&mutable-type=sdmf",
1286                       self.NEWFILE_CONTENTS * 300000))
1287         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1288         d.addCallback(_got_json, "sdmf")
1289         return d
1290
1291     def test_GET_FILEURL_json_mdmf_extensions(self):
1292         # A GET invoked against a URL that includes an MDMF cap with
1293         # extensions should fetch the same JSON information as a GET
1294         # invoked against a bare cap.
1295         self._quux_txt_uri = "%s:3:131073" % self._quux_txt_uri
1296         self._quux_txt_readonly_uri = "%s:3:131073" % self._quux_txt_readonly_uri
1297         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1298         d.addCallback(self.failUnlessIsQuuxJSON)
1299         return d
1300
1301     def test_GET_FILEURL_json_mdmf_bare_cap(self):
1302         elements = self._quux_txt_uri.split(":")
1303         self.failUnlessEqual(len(elements), 6)
1304
1305         quux_uri = ":".join(elements[:len(elements) - 2])
1306         # so failUnlessIsQuuxJSON will work.
1307         self._quux_txt_uri = quux_uri
1308
1309         # we need to alter the readonly URI in the same way, again so
1310         # failUnlessIsQuuxJSON will work
1311         elements = self._quux_txt_readonly_uri.split(":")
1312         self.failUnlessEqual(len(elements), 6)
1313         quux_ro_uri = ":".join(elements[:len(elements) - 2])
1314         self._quux_txt_readonly_uri = quux_ro_uri
1315
1316         base = "/uri/%s?t=json" % urllib.quote(quux_uri)
1317         d = self.GET(base)
1318         d.addCallback(self.failUnlessIsQuuxJSON)
1319         return d
1320
1321     def test_GET_FILEURL_json_mdmf_bare_readonly_cap(self):
1322         elements = self._quux_txt_readonly_uri.split(":")
1323         self.failUnlessEqual(len(elements), 6)
1324
1325         quux_readonly_uri = ":".join(elements[:len(elements) - 2])
1326         # so failUnlessIsQuuxJSON will work
1327         self._quux_txt_readonly_uri = quux_readonly_uri
1328         base = "/uri/%s?t=json" % quux_readonly_uri
1329         d = self.GET(base)
1330         # XXX: We may need to make a method that knows how to check for
1331         # readonly JSON, or else alter that one so that it knows how to
1332         # do that.
1333         d.addCallback(self.failUnlessIsQuuxJSON, readonly=True)
1334         return d
1335
1336     def test_GET_FILEURL_json_mdmf(self):
1337         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1338         d.addCallback(self.failUnlessIsQuuxJSON)
1339         return d
1340
1341     def test_GET_FILEURL_json_missing(self):
1342         d = self.GET(self.public_url + "/foo/missing?json")
1343         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1344         return d
1345
1346     def test_GET_FILEURL_uri(self):
1347         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1348         def _check(res):
1349             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1350         d.addCallback(_check)
1351         d.addCallback(lambda res:
1352                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1353         def _check2(res):
1354             # for now, for files, uris and readonly-uris are the same
1355             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1356         d.addCallback(_check2)
1357         return d
1358
1359     def test_GET_FILEURL_badtype(self):
1360         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1361                                  "bad t=bogus",
1362                                  self.GET,
1363                                  self.public_url + "/foo/bar.txt?t=bogus")
1364         return d
1365
1366     def test_CSS_FILE(self):
1367         d = self.GET("/tahoe_css", followRedirect=True)
1368         def _check(res):
1369             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1370             self.failUnless(CSS_STYLE.search(res), res)
1371         d.addCallback(_check)
1372         return d
1373
1374     def test_GET_FILEURL_uri_missing(self):
1375         d = self.GET(self.public_url + "/foo/missing?t=uri")
1376         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1377         return d
1378
1379     def test_GET_DIRECTORY_html(self):
1380         d = self.GET(self.public_url + "/foo", followRedirect=True)
1381         def _check(res):
1382             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1383             # These are radio buttons that allow a user to toggle
1384             # whether a particular mutable file is SDMF or MDMF.
1385             self.failUnlessIn("mutable-type-mdmf", res)
1386             self.failUnlessIn("mutable-type-sdmf", res)
1387             # Similarly, these toggle whether a particular directory
1388             # should be MDMF or SDMF.
1389             self.failUnlessIn("mutable-directory-mdmf", res)
1390             self.failUnlessIn("mutable-directory-sdmf", res)
1391             self.failUnlessIn("quux", res)
1392         d.addCallback(_check)
1393         return d
1394
1395     def test_GET_root_html(self):
1396         # make sure that we have the option to upload an unlinked
1397         # mutable file in SDMF and MDMF formats.
1398         d = self.GET("/")
1399         def _got_html(html):
1400             # These are radio buttons that allow the user to toggle
1401             # whether a particular mutable file is MDMF or SDMF.
1402             self.failUnlessIn("mutable-type-mdmf", html)
1403             self.failUnlessIn("mutable-type-sdmf", html)
1404             # We should also have the ability to create a mutable directory.
1405             self.failUnlessIn("mkdir", html)
1406             # ...and we should have the ability to say whether that's an
1407             # MDMF or SDMF directory
1408             self.failUnlessIn("mutable-directory-mdmf", html)
1409             self.failUnlessIn("mutable-directory-sdmf", html)
1410         d.addCallback(_got_html)
1411         return d
1412
1413     def test_mutable_type_defaults(self):
1414         # The checked="checked" attribute of the inputs corresponding to
1415         # the mutable-type parameter should change as expected with the
1416         # value configured in tahoe.cfg.
1417         #
1418         # By default, the value configured with the client is
1419         # SDMF_VERSION, so that should be checked.
1420         assert self.s.mutable_file_default == SDMF_VERSION
1421
1422         d = self.GET("/")
1423         def _got_html(html, value):
1424             i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1425             self.failUnlessIn(i % value, html)
1426         d.addCallback(_got_html, "sdmf")
1427         d.addCallback(lambda ignored:
1428             self.GET(self.public_url + "/foo", followRedirect=True))
1429         d.addCallback(_got_html, "sdmf")
1430         # Now switch the configuration value to MDMF. The MDMF radio
1431         # buttons should now be checked on these pages.
1432         def _swap_values(ignored):
1433             self.s.mutable_file_default = MDMF_VERSION
1434         d.addCallback(_swap_values)
1435         d.addCallback(lambda ignored: self.GET("/"))
1436         d.addCallback(_got_html, "mdmf")
1437         d.addCallback(lambda ignored:
1438             self.GET(self.public_url + "/foo", followRedirect=True))
1439         d.addCallback(_got_html, "mdmf")
1440         return d
1441
1442     def test_GET_DIRURL(self):
1443         # the addSlash means we get a redirect here
1444         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1445         ROOT = "../../.."
1446         d = self.GET(self.public_url + "/foo", followRedirect=True)
1447         def _check(res):
1448             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1449                             in res, res)
1450             # the FILE reference points to a URI, but it should end in bar.txt
1451             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1452                        (ROOT, urllib.quote(self._bar_txt_uri)))
1453             get_bar = "".join([r'<td>FILE</td>',
1454                                r'\s+<td>',
1455                                r'<a href="%s">bar.txt</a>' % bar_url,
1456                                r'</td>',
1457                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1458                                ])
1459             self.failUnless(re.search(get_bar, res), res)
1460             for label in ['unlink', 'rename']:
1461                 for line in res.split("\n"):
1462                     # find the line that contains the relevant button for bar.txt
1463                     if ("form action" in line and
1464                         ('value="%s"' % (label,)) in line and
1465                         'value="bar.txt"' in line):
1466                         # the form target should use a relative URL
1467                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1468                         self.failUnlessIn('action="%s"' % foo_url, line)
1469                         # and the when_done= should too
1470                         #done_url = urllib.quote(???)
1471                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1472
1473                         # 'unlink' needs to use POST because it directly has a side effect
1474                         if label == 'unlink':
1475                             self.failUnlessIn('method="post"', line)
1476                         break
1477                 else:
1478                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1479
1480             # the DIR reference just points to a URI
1481             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1482             get_sub = ((r'<td>DIR</td>')
1483                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1484             self.failUnless(re.search(get_sub, res), res)
1485         d.addCallback(_check)
1486
1487         # look at a readonly directory
1488         d.addCallback(lambda res:
1489                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1490         def _check2(res):
1491             self.failUnless("(read-only)" in res, res)
1492             self.failIf("Upload a file" in res, res)
1493         d.addCallback(_check2)
1494
1495         # and at a directory that contains a readonly directory
1496         d.addCallback(lambda res:
1497                       self.GET(self.public_url, followRedirect=True))
1498         def _check3(res):
1499             self.failUnless(re.search('<td>DIR-RO</td>'
1500                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1501         d.addCallback(_check3)
1502
1503         # and an empty directory
1504         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1505         def _check4(res):
1506             self.failUnless("directory is empty" in res, res)
1507             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1508             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1509         d.addCallback(_check4)
1510
1511         # and at a literal directory
1512         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1513         d.addCallback(lambda res:
1514                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1515         def _check5(res):
1516             self.failUnless('(immutable)' in res, res)
1517             self.failUnless(re.search('<td>FILE</td>'
1518                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1519         d.addCallback(_check5)
1520         return d
1521
1522     def test_GET_DIRURL_badtype(self):
1523         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1524                                  400, "Bad Request",
1525                                  "bad t=bogus",
1526                                  self.GET,
1527                                  self.public_url + "/foo?t=bogus")
1528         return d
1529
1530     def test_GET_DIRURL_json(self):
1531         d = self.GET(self.public_url + "/foo?t=json")
1532         d.addCallback(self.failUnlessIsFooJSON)
1533         return d
1534
1535     def test_GET_DIRURL_json_mutable_type(self):
1536         d = self.PUT(self.public_url + \
1537                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
1538                      self.NEWFILE_CONTENTS * 300000)
1539         d.addCallback(lambda ignored:
1540             self.PUT(self.public_url + \
1541                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
1542                      self.NEWFILE_CONTENTS * 300000))
1543         # Now we have an MDMF and SDMF file in the directory. If we GET
1544         # its JSON, we should see their encodings.
1545         d.addCallback(lambda ignored:
1546             self.GET(self.public_url + "/foo?t=json"))
1547         def _got_json(json):
1548             data = simplejson.loads(json)
1549             assert data[0] == "dirnode"
1550
1551             data = data[1]
1552             kids = data['children']
1553
1554             mdmf_data = kids['mdmf.txt'][1]
1555             self.failUnlessIn("mutable-type", mdmf_data)
1556             self.failUnlessEqual(mdmf_data['mutable-type'], "mdmf")
1557
1558             sdmf_data = kids['sdmf.txt'][1]
1559             self.failUnlessIn("mutable-type", sdmf_data)
1560             self.failUnlessEqual(sdmf_data['mutable-type'], "sdmf")
1561         d.addCallback(_got_json)
1562         return d
1563
1564
1565     def test_POST_DIRURL_manifest_no_ophandle(self):
1566         d = self.shouldFail2(error.Error,
1567                              "test_POST_DIRURL_manifest_no_ophandle",
1568                              "400 Bad Request",
1569                              "slow operation requires ophandle=",
1570                              self.POST, self.public_url, t="start-manifest")
1571         return d
1572
1573     def test_POST_DIRURL_manifest(self):
1574         d = defer.succeed(None)
1575         def getman(ignored, output):
1576             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1577                           followRedirect=True)
1578             d.addCallback(self.wait_for_operation, "125")
1579             d.addCallback(self.get_operation_results, "125", output)
1580             return d
1581         d.addCallback(getman, None)
1582         def _got_html(manifest):
1583             self.failUnless("Manifest of SI=" in manifest)
1584             self.failUnless("<td>sub</td>" in manifest)
1585             self.failUnless(self._sub_uri in manifest)
1586             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1587         d.addCallback(_got_html)
1588
1589         # both t=status and unadorned GET should be identical
1590         d.addCallback(lambda res: self.GET("/operations/125"))
1591         d.addCallback(_got_html)
1592
1593         d.addCallback(getman, "html")
1594         d.addCallback(_got_html)
1595         d.addCallback(getman, "text")
1596         def _got_text(manifest):
1597             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1598             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1599         d.addCallback(_got_text)
1600         d.addCallback(getman, "JSON")
1601         def _got_json(res):
1602             data = res["manifest"]
1603             got = {}
1604             for (path_list, cap) in data:
1605                 got[tuple(path_list)] = cap
1606             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1607             self.failUnless((u"sub",u"baz.txt") in got)
1608             self.failUnless("finished" in res)
1609             self.failUnless("origin" in res)
1610             self.failUnless("storage-index" in res)
1611             self.failUnless("verifycaps" in res)
1612             self.failUnless("stats" in res)
1613         d.addCallback(_got_json)
1614         return d
1615
1616     def test_POST_DIRURL_deepsize_no_ophandle(self):
1617         d = self.shouldFail2(error.Error,
1618                              "test_POST_DIRURL_deepsize_no_ophandle",
1619                              "400 Bad Request",
1620                              "slow operation requires ophandle=",
1621                              self.POST, self.public_url, t="start-deep-size")
1622         return d
1623
1624     def test_POST_DIRURL_deepsize(self):
1625         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1626                       followRedirect=True)
1627         d.addCallback(self.wait_for_operation, "126")
1628         d.addCallback(self.get_operation_results, "126", "json")
1629         def _got_json(data):
1630             self.failUnlessReallyEqual(data["finished"], True)
1631             size = data["size"]
1632             self.failUnless(size > 1000)
1633         d.addCallback(_got_json)
1634         d.addCallback(self.get_operation_results, "126", "text")
1635         def _got_text(res):
1636             mo = re.search(r'^size: (\d+)$', res, re.M)
1637             self.failUnless(mo, res)
1638             size = int(mo.group(1))
1639             # with directories, the size varies.
1640             self.failUnless(size > 1000)
1641         d.addCallback(_got_text)
1642         return d
1643
1644     def test_POST_DIRURL_deepstats_no_ophandle(self):
1645         d = self.shouldFail2(error.Error,
1646                              "test_POST_DIRURL_deepstats_no_ophandle",
1647                              "400 Bad Request",
1648                              "slow operation requires ophandle=",
1649                              self.POST, self.public_url, t="start-deep-stats")
1650         return d
1651
1652     def test_POST_DIRURL_deepstats(self):
1653         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1654                       followRedirect=True)
1655         d.addCallback(self.wait_for_operation, "127")
1656         d.addCallback(self.get_operation_results, "127", "json")
1657         def _got_json(stats):
1658             expected = {"count-immutable-files": 3,
1659                         "count-mutable-files": 2,
1660                         "count-literal-files": 0,
1661                         "count-files": 5,
1662                         "count-directories": 3,
1663                         "size-immutable-files": 57,
1664                         "size-literal-files": 0,
1665                         #"size-directories": 1912, # varies
1666                         #"largest-directory": 1590,
1667                         "largest-directory-children": 7,
1668                         "largest-immutable-file": 19,
1669                         }
1670             for k,v in expected.iteritems():
1671                 self.failUnlessReallyEqual(stats[k], v,
1672                                            "stats[%s] was %s, not %s" %
1673                                            (k, stats[k], v))
1674             self.failUnlessReallyEqual(stats["size-files-histogram"],
1675                                        [ [11, 31, 3] ])
1676         d.addCallback(_got_json)
1677         return d
1678
1679     def test_POST_DIRURL_stream_manifest(self):
1680         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1681         def _check(res):
1682             self.failUnless(res.endswith("\n"))
1683             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1684             self.failUnlessReallyEqual(len(units), 9)
1685             self.failUnlessEqual(units[-1]["type"], "stats")
1686             first = units[0]
1687             self.failUnlessEqual(first["path"], [])
1688             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1689             self.failUnlessEqual(first["type"], "directory")
1690             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1691             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1692             self.failIfEqual(baz["storage-index"], None)
1693             self.failIfEqual(baz["verifycap"], None)
1694             self.failIfEqual(baz["repaircap"], None)
1695             # XXX: Add quux and baz to this test.
1696             return
1697         d.addCallback(_check)
1698         return d
1699
1700     def test_GET_DIRURL_uri(self):
1701         d = self.GET(self.public_url + "/foo?t=uri")
1702         def _check(res):
1703             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1704         d.addCallback(_check)
1705         return d
1706
1707     def test_GET_DIRURL_readonly_uri(self):
1708         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1709         def _check(res):
1710             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1711         d.addCallback(_check)
1712         return d
1713
1714     def test_PUT_NEWDIRURL(self):
1715         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1716         d.addCallback(lambda res:
1717                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1718         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1719         d.addCallback(self.failUnlessNodeKeysAre, [])
1720         return d
1721
1722     def test_PUT_NEWDIRURL_mdmf(self):
1723         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1724         d.addCallback(lambda res:
1725                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1726         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1727         d.addCallback(lambda node:
1728             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1729         return d
1730
1731     def test_PUT_NEWDIRURL_sdmf(self):
1732         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf",
1733                      "")
1734         d.addCallback(lambda res:
1735                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1736         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1737         d.addCallback(lambda node:
1738             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1739         return d
1740
1741     def test_PUT_NEWDIRURL_bad_mutable_type(self):
1742         return self.shouldHTTPError("test bad mutable type",
1743                              400, "Bad Request", "Unknown type: foo",
1744                              self.PUT, self.public_url + \
1745                              "/foo/newdir=?t=mkdir&mutable-type=foo", "")
1746
1747     def test_POST_NEWDIRURL(self):
1748         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1749         d.addCallback(lambda res:
1750                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1751         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1752         d.addCallback(self.failUnlessNodeKeysAre, [])
1753         return d
1754
1755     def test_POST_NEWDIRURL_mdmf(self):
1756         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1757         d.addCallback(lambda res:
1758                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1759         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1760         d.addCallback(lambda node:
1761             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1762         return d
1763
1764     def test_POST_NEWDIRURL_sdmf(self):
1765         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf", "")
1766         d.addCallback(lambda res:
1767             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1768         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1769         d.addCallback(lambda node:
1770             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1771         return d
1772
1773     def test_POST_NEWDIRURL_bad_mutable_type(self):
1774         return self.shouldHTTPError("test bad mutable type",
1775                                     400, "Bad Request", "Unknown type: foo",
1776                                     self.POST2, self.public_url + \
1777                                     "/foo/newdir?t=mkdir&mutable-type=foo", "")
1778
1779     def test_POST_NEWDIRURL_emptyname(self):
1780         # an empty pathname component (i.e. a double-slash) is disallowed
1781         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1782                              "400 Bad Request",
1783                              "The webapi does not allow empty pathname components, i.e. a double slash",
1784                              self.POST, self.public_url + "//?t=mkdir")
1785         return d
1786
1787     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1788         (newkids, caps) = self._create_initial_children()
1789         query = "/foo/newdir?t=mkdir-with-children"
1790         if version == MDMF_VERSION:
1791             query += "&mutable-type=mdmf"
1792         elif version == SDMF_VERSION:
1793             query += "&mutable-type=sdmf"
1794         else:
1795             version = SDMF_VERSION # for later
1796         d = self.POST2(self.public_url + query,
1797                        simplejson.dumps(newkids))
1798         def _check(uri):
1799             n = self.s.create_node_from_uri(uri.strip())
1800             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1801             self.failUnlessEqual(n._node.get_version(), version)
1802             d2.addCallback(lambda ign:
1803                            self.failUnlessROChildURIIs(n, u"child-imm",
1804                                                        caps['filecap1']))
1805             d2.addCallback(lambda ign:
1806                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1807                                                        caps['filecap2']))
1808             d2.addCallback(lambda ign:
1809                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1810                                                        caps['filecap3']))
1811             d2.addCallback(lambda ign:
1812                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1813                                                        caps['unknown_rocap']))
1814             d2.addCallback(lambda ign:
1815                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1816                                                        caps['unknown_rwcap']))
1817             d2.addCallback(lambda ign:
1818                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1819                                                        caps['unknown_immcap']))
1820             d2.addCallback(lambda ign:
1821                            self.failUnlessRWChildURIIs(n, u"dirchild",
1822                                                        caps['dircap']))
1823             d2.addCallback(lambda ign:
1824                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1825                                                        caps['litdircap']))
1826             d2.addCallback(lambda ign:
1827                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1828                                                        caps['emptydircap']))
1829             return d2
1830         d.addCallback(_check)
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1833         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1834         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1835         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1836         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1837         return d
1838
1839     def test_POST_NEWDIRURL_initial_children(self):
1840         return self._do_POST_NEWDIRURL_initial_children_test()
1841
1842     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1843         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1844
1845     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1846         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1847
1848     def test_POST_NEWDIRURL_initial_children_bad_mutable_type(self):
1849         (newkids, caps) = self._create_initial_children()
1850         return self.shouldHTTPError("test bad mutable type",
1851                                     400, "Bad Request", "Unknown type: foo",
1852                                     self.POST2, self.public_url + \
1853                                     "/foo/newdir?t=mkdir-with-children&mutable-type=foo",
1854                                     simplejson.dumps(newkids))
1855
1856     def test_POST_NEWDIRURL_immutable(self):
1857         (newkids, caps) = self._create_immutable_children()
1858         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1859                        simplejson.dumps(newkids))
1860         def _check(uri):
1861             n = self.s.create_node_from_uri(uri.strip())
1862             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1863             d2.addCallback(lambda ign:
1864                            self.failUnlessROChildURIIs(n, u"child-imm",
1865                                                        caps['filecap1']))
1866             d2.addCallback(lambda ign:
1867                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1868                                                        caps['unknown_immcap']))
1869             d2.addCallback(lambda ign:
1870                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1871                                                        caps['immdircap']))
1872             d2.addCallback(lambda ign:
1873                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1874                                                        caps['litdircap']))
1875             d2.addCallback(lambda ign:
1876                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1877                                                        caps['emptydircap']))
1878             return d2
1879         d.addCallback(_check)
1880         d.addCallback(lambda res:
1881                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1882         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1883         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1884         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1885         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1886         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1887         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1888         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1889         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1890         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1891         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1892         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1893         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1894         d.addErrback(self.explain_web_error)
1895         return d
1896
1897     def test_POST_NEWDIRURL_immutable_bad(self):
1898         (newkids, caps) = self._create_initial_children()
1899         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1900                              "400 Bad Request",
1901                              "needed to be immutable but was not",
1902                              self.POST2,
1903                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1904                              simplejson.dumps(newkids))
1905         return d
1906
1907     def test_PUT_NEWDIRURL_exists(self):
1908         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1909         d.addCallback(lambda res:
1910                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1911         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1912         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1913         return d
1914
1915     def test_PUT_NEWDIRURL_blocked(self):
1916         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1917                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1918                              self.PUT,
1919                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1920         d.addCallback(lambda res:
1921                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1922         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1923         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1924         return d
1925
1926     def test_PUT_NEWDIRURL_mkdir_p(self):
1927         d = defer.succeed(None)
1928         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1929         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1930         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1931         def mkdir_p(mkpnode):
1932             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1933             d = self.POST(url)
1934             def made_subsub(ssuri):
1935                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1936                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1937                 d = self.POST(url)
1938                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1939                 return d
1940             d.addCallback(made_subsub)
1941             return d
1942         d.addCallback(mkdir_p)
1943         return d
1944
1945     def test_PUT_NEWDIRURL_mkdirs(self):
1946         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1947         d.addCallback(lambda res:
1948                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1949         d.addCallback(lambda res:
1950                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1951         d.addCallback(lambda res:
1952                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1953         d.addCallback(self.failUnlessNodeKeysAre, [])
1954         return d
1955
1956     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1957         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=mdmf", "")
1958         d.addCallback(lambda ignored:
1959             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1960         d.addCallback(lambda ignored:
1961             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1962         d.addCallback(lambda ignored:
1963             self._foo_node.get_child_at_path(u"subdir"))
1964         def _got_subdir(subdir):
1965             # XXX: What we want?
1966             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1967             self.failUnlessNodeHasChild(subdir, u"newdir")
1968             return subdir.get_child_at_path(u"newdir")
1969         d.addCallback(_got_subdir)
1970         d.addCallback(lambda newdir:
1971             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1972         return d
1973
1974     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1975         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=sdmf", "")
1976         d.addCallback(lambda ignored:
1977             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1978         d.addCallback(lambda ignored:
1979             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1980         d.addCallback(lambda ignored:
1981             self._foo_node.get_child_at_path(u"subdir"))
1982         def _got_subdir(subdir):
1983             # XXX: What we want?
1984             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1985             self.failUnlessNodeHasChild(subdir, u"newdir")
1986             return subdir.get_child_at_path(u"newdir")
1987         d.addCallback(_got_subdir)
1988         d.addCallback(lambda newdir:
1989             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1990         return d
1991
1992     def test_PUT_NEWDIRURL_mkdirs_bad_mutable_type(self):
1993         return self.shouldHTTPError("test bad mutable type",
1994                                     400, "Bad Request", "Unknown type: foo",
1995                                     self.PUT, self.public_url + \
1996                                     "/foo/subdir/newdir?t=mkdir&mutable-type=foo",
1997                                     "")
1998
1999     def test_DELETE_DIRURL(self):
2000         d = self.DELETE(self.public_url + "/foo")
2001         d.addCallback(lambda res:
2002                       self.failIfNodeHasChild(self.public_root, u"foo"))
2003         return d
2004
2005     def test_DELETE_DIRURL_missing(self):
2006         d = self.DELETE(self.public_url + "/foo/missing")
2007         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2008         d.addCallback(lambda res:
2009                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2010         return d
2011
2012     def test_DELETE_DIRURL_missing2(self):
2013         d = self.DELETE(self.public_url + "/missing")
2014         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2015         return d
2016
2017     def dump_root(self):
2018         print "NODEWALK"
2019         w = webish.DirnodeWalkerMixin()
2020         def visitor(childpath, childnode, metadata):
2021             print childpath
2022         d = w.walk(self.public_root, visitor)
2023         return d
2024
2025     def failUnlessNodeKeysAre(self, node, expected_keys):
2026         for k in expected_keys:
2027             assert isinstance(k, unicode)
2028         d = node.list()
2029         def _check(children):
2030             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2031         d.addCallback(_check)
2032         return d
2033     def failUnlessNodeHasChild(self, node, name):
2034         assert isinstance(name, unicode)
2035         d = node.list()
2036         def _check(children):
2037             self.failUnless(name in children)
2038         d.addCallback(_check)
2039         return d
2040     def failIfNodeHasChild(self, node, name):
2041         assert isinstance(name, unicode)
2042         d = node.list()
2043         def _check(children):
2044             self.failIf(name in children)
2045         d.addCallback(_check)
2046         return d
2047
2048     def failUnlessChildContentsAre(self, node, name, expected_contents):
2049         assert isinstance(name, unicode)
2050         d = node.get_child_at_path(name)
2051         d.addCallback(lambda node: download_to_data(node))
2052         def _check(contents):
2053             self.failUnlessReallyEqual(contents, expected_contents)
2054         d.addCallback(_check)
2055         return d
2056
2057     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2058         assert isinstance(name, unicode)
2059         d = node.get_child_at_path(name)
2060         d.addCallback(lambda node: node.download_best_version())
2061         def _check(contents):
2062             self.failUnlessReallyEqual(contents, expected_contents)
2063         d.addCallback(_check)
2064         return d
2065
2066     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2067         assert isinstance(name, unicode)
2068         d = node.get_child_at_path(name)
2069         def _check(child):
2070             self.failUnless(child.is_unknown() or not child.is_readonly())
2071             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2072             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2073             expected_ro_uri = self._make_readonly(expected_uri)
2074             if expected_ro_uri:
2075                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2076         d.addCallback(_check)
2077         return d
2078
2079     def failUnlessROChildURIIs(self, node, name, expected_uri):
2080         assert isinstance(name, unicode)
2081         d = node.get_child_at_path(name)
2082         def _check(child):
2083             self.failUnless(child.is_unknown() or child.is_readonly())
2084             self.failUnlessReallyEqual(child.get_write_uri(), None)
2085             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2086             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2087         d.addCallback(_check)
2088         return d
2089
2090     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2091         assert isinstance(name, unicode)
2092         d = node.get_child_at_path(name)
2093         def _check(child):
2094             self.failUnless(child.is_unknown() or not child.is_readonly())
2095             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2096             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2097             expected_ro_uri = self._make_readonly(got_uri)
2098             if expected_ro_uri:
2099                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2100         d.addCallback(_check)
2101         return d
2102
2103     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2104         assert isinstance(name, unicode)
2105         d = node.get_child_at_path(name)
2106         def _check(child):
2107             self.failUnless(child.is_unknown() or child.is_readonly())
2108             self.failUnlessReallyEqual(child.get_write_uri(), None)
2109             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2110             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2111         d.addCallback(_check)
2112         return d
2113
2114     def failUnlessCHKURIHasContents(self, got_uri, contents):
2115         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2116
2117     def test_POST_upload(self):
2118         d = self.POST(self.public_url + "/foo", t="upload",
2119                       file=("new.txt", self.NEWFILE_CONTENTS))
2120         fn = self._foo_node
2121         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2122         d.addCallback(lambda res:
2123                       self.failUnlessChildContentsAre(fn, u"new.txt",
2124                                                       self.NEWFILE_CONTENTS))
2125         return d
2126
2127     def test_POST_upload_unicode(self):
2128         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2129         d = self.POST(self.public_url + "/foo", t="upload",
2130                       file=(filename, self.NEWFILE_CONTENTS))
2131         fn = self._foo_node
2132         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2133         d.addCallback(lambda res:
2134                       self.failUnlessChildContentsAre(fn, filename,
2135                                                       self.NEWFILE_CONTENTS))
2136         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2137         d.addCallback(lambda res: self.GET(target_url))
2138         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2139                                                                   self.NEWFILE_CONTENTS,
2140                                                                   contents))
2141         return d
2142
2143     def test_POST_upload_unicode_named(self):
2144         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2145         d = self.POST(self.public_url + "/foo", t="upload",
2146                       name=filename,
2147                       file=("overridden", self.NEWFILE_CONTENTS))
2148         fn = self._foo_node
2149         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2150         d.addCallback(lambda res:
2151                       self.failUnlessChildContentsAre(fn, filename,
2152                                                       self.NEWFILE_CONTENTS))
2153         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2154         d.addCallback(lambda res: self.GET(target_url))
2155         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2156                                                                   self.NEWFILE_CONTENTS,
2157                                                                   contents))
2158         return d
2159
2160     def test_POST_upload_no_link(self):
2161         d = self.POST("/uri", t="upload",
2162                       file=("new.txt", self.NEWFILE_CONTENTS))
2163         def _check_upload_results(page):
2164             # this should be a page which describes the results of the upload
2165             # that just finished.
2166             self.failUnless("Upload Results:" in page)
2167             self.failUnless("URI:" in page)
2168             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2169             mo = uri_re.search(page)
2170             self.failUnless(mo, page)
2171             new_uri = mo.group(1)
2172             return new_uri
2173         d.addCallback(_check_upload_results)
2174         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2175         return d
2176
2177     def test_POST_upload_no_link_whendone(self):
2178         d = self.POST("/uri", t="upload", when_done="/",
2179                       file=("new.txt", self.NEWFILE_CONTENTS))
2180         d.addBoth(self.shouldRedirect, "/")
2181         return d
2182
2183     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2184         d = defer.maybeDeferred(callable, *args, **kwargs)
2185         def done(res):
2186             if isinstance(res, failure.Failure):
2187                 res.trap(error.PageRedirect)
2188                 statuscode = res.value.status
2189                 target = res.value.location
2190                 return checker(statuscode, target)
2191             self.fail("%s: callable was supposed to redirect, not return '%s'"
2192                       % (which, res))
2193         d.addBoth(done)
2194         return d
2195
2196     def test_POST_upload_no_link_whendone_results(self):
2197         def check(statuscode, target):
2198             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2199             self.failUnless(target.startswith(self.webish_url), target)
2200             return client.getPage(target, method="GET")
2201         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2202                                  check,
2203                                  self.POST, "/uri", t="upload",
2204                                  when_done="/uri/%(uri)s",
2205                                  file=("new.txt", self.NEWFILE_CONTENTS))
2206         d.addCallback(lambda res:
2207                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2208         return d
2209
2210     def test_POST_upload_no_link_mutable(self):
2211         d = self.POST("/uri", t="upload", mutable="true",
2212                       file=("new.txt", self.NEWFILE_CONTENTS))
2213         def _check(filecap):
2214             filecap = filecap.strip()
2215             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2216             self.filecap = filecap
2217             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2218             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2219             n = self.s.create_node_from_uri(filecap)
2220             return n.download_best_version()
2221         d.addCallback(_check)
2222         def _check2(data):
2223             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2224             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2225         d.addCallback(_check2)
2226         def _check3(data):
2227             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2228             return self.GET("/file/%s" % urllib.quote(self.filecap))
2229         d.addCallback(_check3)
2230         def _check4(data):
2231             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2232         d.addCallback(_check4)
2233         return d
2234
2235     def test_POST_upload_no_link_mutable_toobig(self):
2236         # The SDMF size limit is no longer in place, so we should be
2237         # able to upload mutable files that are as large as we want them
2238         # to be.
2239         d = self.POST("/uri", t="upload", mutable="true",
2240                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2241         return d
2242
2243
2244     def test_POST_upload_mutable_type_unlinked(self):
2245         d = self.POST("/uri?t=upload&mutable=true&mutable-type=sdmf",
2246                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2247         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2248         def _got_json(json, version):
2249             data = simplejson.loads(json)
2250             data = data[1]
2251
2252             self.failUnlessIn("mutable-type", data)
2253             self.failUnlessEqual(data['mutable-type'], version)
2254         d.addCallback(_got_json, "sdmf")
2255         d.addCallback(lambda ignored:
2256             self.POST("/uri?t=upload&mutable=true&mutable-type=mdmf",
2257                       file=('mdmf.txt', self.NEWFILE_CONTENTS * 300000)))
2258         def _got_filecap(filecap):
2259             self.failUnless(filecap.startswith("URI:MDMF"))
2260             return filecap
2261         d.addCallback(_got_filecap)
2262         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2263         d.addCallback(_got_json, "mdmf")
2264         return d
2265
2266     def test_POST_upload_mutable_type_unlinked_bad_mutable_type(self):
2267         return self.shouldHTTPError("test bad mutable type",
2268                                     400, "Bad Request", "Unknown type: foo",
2269                                     self.POST,
2270                                     "/uri?5=upload&mutable=true&mutable-type=foo",
2271                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2272
2273     def test_POST_upload_mutable_type(self):
2274         d = self.POST(self.public_url + \
2275                       "/foo?t=upload&mutable=true&mutable-type=sdmf",
2276                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2277         fn = self._foo_node
2278         def _got_cap(filecap, filename):
2279             filenameu = unicode(filename)
2280             self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2281             return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2282         def _got_mdmf_cap(filecap):
2283             self.failUnless(filecap.startswith("URI:MDMF"))
2284             return filecap
2285         d.addCallback(_got_cap, "sdmf.txt")
2286         def _got_json(json, version):
2287             data = simplejson.loads(json)
2288             data = data[1]
2289
2290             self.failUnlessIn("mutable-type", data)
2291             self.failUnlessEqual(data['mutable-type'], version)
2292         d.addCallback(_got_json, "sdmf")
2293         d.addCallback(lambda ignored:
2294             self.POST(self.public_url + \
2295                       "/foo?t=upload&mutable=true&mutable-type=mdmf",
2296                       file=("mdmf.txt", self.NEWFILE_CONTENTS * 300000)))
2297         d.addCallback(_got_mdmf_cap)
2298         d.addCallback(_got_cap, "mdmf.txt")
2299         d.addCallback(_got_json, "mdmf")
2300         return d
2301
2302     def test_POST_upload_bad_mutable_type(self):
2303         return self.shouldHTTPError("test bad mutable type",
2304                                     400, "Bad Request", "Unknown type: foo",
2305                                     self.POST, self.public_url + \
2306                                     "/foo?t=upload&mutable=true&mutable-type=foo",
2307                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2308
2309     def test_POST_upload_mutable(self):
2310         # this creates a mutable file
2311         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2312                       file=("new.txt", self.NEWFILE_CONTENTS))
2313         fn = self._foo_node
2314         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2315         d.addCallback(lambda res:
2316                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2317                                                              self.NEWFILE_CONTENTS))
2318         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2319         def _got(newnode):
2320             self.failUnless(IMutableFileNode.providedBy(newnode))
2321             self.failUnless(newnode.is_mutable())
2322             self.failIf(newnode.is_readonly())
2323             self._mutable_node = newnode
2324             self._mutable_uri = newnode.get_uri()
2325         d.addCallback(_got)
2326
2327         # now upload it again and make sure that the URI doesn't change
2328         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2329         d.addCallback(lambda res:
2330                       self.POST(self.public_url + "/foo", t="upload",
2331                                 mutable="true",
2332                                 file=("new.txt", NEWER_CONTENTS)))
2333         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2334         d.addCallback(lambda res:
2335                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2336                                                              NEWER_CONTENTS))
2337         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2338         def _got2(newnode):
2339             self.failUnless(IMutableFileNode.providedBy(newnode))
2340             self.failUnless(newnode.is_mutable())
2341             self.failIf(newnode.is_readonly())
2342             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2343         d.addCallback(_got2)
2344
2345         # upload a second time, using PUT instead of POST
2346         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2347         d.addCallback(lambda res:
2348                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2349         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2350         d.addCallback(lambda res:
2351                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2352                                                              NEW2_CONTENTS))
2353
2354         # finally list the directory, since mutable files are displayed
2355         # slightly differently
2356
2357         d.addCallback(lambda res:
2358                       self.GET(self.public_url + "/foo/",
2359                                followRedirect=True))
2360         def _check_page(res):
2361             # TODO: assert more about the contents
2362             self.failUnless("SSK" in res)
2363             return res
2364         d.addCallback(_check_page)
2365
2366         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2367         def _got3(newnode):
2368             self.failUnless(IMutableFileNode.providedBy(newnode))
2369             self.failUnless(newnode.is_mutable())
2370             self.failIf(newnode.is_readonly())
2371             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2372         d.addCallback(_got3)
2373
2374         # look at the JSON form of the enclosing directory
2375         d.addCallback(lambda res:
2376                       self.GET(self.public_url + "/foo/?t=json",
2377                                followRedirect=True))
2378         def _check_page_json(res):
2379             parsed = simplejson.loads(res)
2380             self.failUnlessEqual(parsed[0], "dirnode")
2381             children = dict( [(unicode(name),value)
2382                               for (name,value)
2383                               in parsed[1]["children"].iteritems()] )
2384             self.failUnless(u"new.txt" in children)
2385             new_json = children[u"new.txt"]
2386             self.failUnlessEqual(new_json[0], "filenode")
2387             self.failUnless(new_json[1]["mutable"])
2388             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2389             ro_uri = self._mutable_node.get_readonly().to_string()
2390             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2391         d.addCallback(_check_page_json)
2392
2393         # and the JSON form of the file
2394         d.addCallback(lambda res:
2395                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2396         def _check_file_json(res):
2397             parsed = simplejson.loads(res)
2398             self.failUnlessEqual(parsed[0], "filenode")
2399             self.failUnless(parsed[1]["mutable"])
2400             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2401             ro_uri = self._mutable_node.get_readonly().to_string()
2402             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2403         d.addCallback(_check_file_json)
2404
2405         # and look at t=uri and t=readonly-uri
2406         d.addCallback(lambda res:
2407                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2408         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2409         d.addCallback(lambda res:
2410                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2411         def _check_ro_uri(res):
2412             ro_uri = self._mutable_node.get_readonly().to_string()
2413             self.failUnlessReallyEqual(res, ro_uri)
2414         d.addCallback(_check_ro_uri)
2415
2416         # make sure we can get to it from /uri/URI
2417         d.addCallback(lambda res:
2418                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2419         d.addCallback(lambda res:
2420                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2421
2422         # and that HEAD computes the size correctly
2423         d.addCallback(lambda res:
2424                       self.HEAD(self.public_url + "/foo/new.txt",
2425                                 return_response=True))
2426         def _got_headers((res, status, headers)):
2427             self.failUnlessReallyEqual(res, "")
2428             self.failUnlessReallyEqual(headers["content-length"][0],
2429                                        str(len(NEW2_CONTENTS)))
2430             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2431         d.addCallback(_got_headers)
2432
2433         # make sure that outdated size limits aren't enforced anymore.
2434         d.addCallback(lambda ignored:
2435             self.POST(self.public_url + "/foo", t="upload",
2436                       mutable="true",
2437                       file=("new.txt",
2438                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2439         d.addErrback(self.dump_error)
2440         return d
2441
2442     def test_POST_upload_mutable_toobig(self):
2443         # SDMF had a size limti that was removed a while ago. MDMF has
2444         # never had a size limit. Test to make sure that we do not
2445         # encounter errors when trying to upload large mutable files,
2446         # since there should be no coded prohibitions regarding large
2447         # mutable files.
2448         d = self.POST(self.public_url + "/foo",
2449                       t="upload", mutable="true",
2450                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2451         return d
2452
2453     def dump_error(self, f):
2454         # if the web server returns an error code (like 400 Bad Request),
2455         # web.client.getPage puts the HTTP response body into the .response
2456         # attribute of the exception object that it gives back. It does not
2457         # appear in the Failure's repr(), so the ERROR that trial displays
2458         # will be rather terse and unhelpful. addErrback this method to the
2459         # end of your chain to get more information out of these errors.
2460         if f.check(error.Error):
2461             print "web.error.Error:"
2462             print f
2463             print f.value.response
2464         return f
2465
2466     def test_POST_upload_replace(self):
2467         d = self.POST(self.public_url + "/foo", t="upload",
2468                       file=("bar.txt", self.NEWFILE_CONTENTS))
2469         fn = self._foo_node
2470         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2471         d.addCallback(lambda res:
2472                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2473                                                       self.NEWFILE_CONTENTS))
2474         return d
2475
2476     def test_POST_upload_no_replace_ok(self):
2477         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2478                       file=("new.txt", self.NEWFILE_CONTENTS))
2479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2480         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2481                                                              self.NEWFILE_CONTENTS))
2482         return d
2483
2484     def test_POST_upload_no_replace_queryarg(self):
2485         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2486                       file=("bar.txt", self.NEWFILE_CONTENTS))
2487         d.addBoth(self.shouldFail, error.Error,
2488                   "POST_upload_no_replace_queryarg",
2489                   "409 Conflict",
2490                   "There was already a child by that name, and you asked me "
2491                   "to not replace it")
2492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2493         d.addCallback(self.failUnlessIsBarDotTxt)
2494         return d
2495
2496     def test_POST_upload_no_replace_field(self):
2497         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2498                       file=("bar.txt", self.NEWFILE_CONTENTS))
2499         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2500                   "409 Conflict",
2501                   "There was already a child by that name, and you asked me "
2502                   "to not replace it")
2503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2504         d.addCallback(self.failUnlessIsBarDotTxt)
2505         return d
2506
2507     def test_POST_upload_whendone(self):
2508         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2509                       file=("new.txt", self.NEWFILE_CONTENTS))
2510         d.addBoth(self.shouldRedirect, "/THERE")
2511         fn = self._foo_node
2512         d.addCallback(lambda res:
2513                       self.failUnlessChildContentsAre(fn, u"new.txt",
2514                                                       self.NEWFILE_CONTENTS))
2515         return d
2516
2517     def test_POST_upload_named(self):
2518         fn = self._foo_node
2519         d = self.POST(self.public_url + "/foo", t="upload",
2520                       name="new.txt", file=self.NEWFILE_CONTENTS)
2521         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2522         d.addCallback(lambda res:
2523                       self.failUnlessChildContentsAre(fn, u"new.txt",
2524                                                       self.NEWFILE_CONTENTS))
2525         return d
2526
2527     def test_POST_upload_named_badfilename(self):
2528         d = self.POST(self.public_url + "/foo", t="upload",
2529                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2530         d.addBoth(self.shouldFail, error.Error,
2531                   "test_POST_upload_named_badfilename",
2532                   "400 Bad Request",
2533                   "name= may not contain a slash",
2534                   )
2535         # make sure that nothing was added
2536         d.addCallback(lambda res:
2537                       self.failUnlessNodeKeysAre(self._foo_node,
2538                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2539                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2540                                                   u"sub"]))
2541         return d
2542
2543     def test_POST_FILEURL_check(self):
2544         bar_url = self.public_url + "/foo/bar.txt"
2545         d = self.POST(bar_url, t="check")
2546         def _check(res):
2547             self.failUnless("Healthy :" in res)
2548         d.addCallback(_check)
2549         redir_url = "http://allmydata.org/TARGET"
2550         def _check2(statuscode, target):
2551             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2552             self.failUnlessReallyEqual(target, redir_url)
2553         d.addCallback(lambda res:
2554                       self.shouldRedirect2("test_POST_FILEURL_check",
2555                                            _check2,
2556                                            self.POST, bar_url,
2557                                            t="check",
2558                                            when_done=redir_url))
2559         d.addCallback(lambda res:
2560                       self.POST(bar_url, t="check", return_to=redir_url))
2561         def _check3(res):
2562             self.failUnless("Healthy :" in res)
2563             self.failUnless("Return to file" in res)
2564             self.failUnless(redir_url in res)
2565         d.addCallback(_check3)
2566
2567         d.addCallback(lambda res:
2568                       self.POST(bar_url, t="check", output="JSON"))
2569         def _check_json(res):
2570             data = simplejson.loads(res)
2571             self.failUnless("storage-index" in data)
2572             self.failUnless(data["results"]["healthy"])
2573         d.addCallback(_check_json)
2574
2575         return d
2576
2577     def test_POST_FILEURL_check_and_repair(self):
2578         bar_url = self.public_url + "/foo/bar.txt"
2579         d = self.POST(bar_url, t="check", repair="true")
2580         def _check(res):
2581             self.failUnless("Healthy :" in res)
2582         d.addCallback(_check)
2583         redir_url = "http://allmydata.org/TARGET"
2584         def _check2(statuscode, target):
2585             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2586             self.failUnlessReallyEqual(target, redir_url)
2587         d.addCallback(lambda res:
2588                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2589                                            _check2,
2590                                            self.POST, bar_url,
2591                                            t="check", repair="true",
2592                                            when_done=redir_url))
2593         d.addCallback(lambda res:
2594                       self.POST(bar_url, t="check", return_to=redir_url))
2595         def _check3(res):
2596             self.failUnless("Healthy :" in res)
2597             self.failUnless("Return to file" in res)
2598             self.failUnless(redir_url in res)
2599         d.addCallback(_check3)
2600         return d
2601
2602     def test_POST_DIRURL_check(self):
2603         foo_url = self.public_url + "/foo/"
2604         d = self.POST(foo_url, t="check")
2605         def _check(res):
2606             self.failUnless("Healthy :" in res, res)
2607         d.addCallback(_check)
2608         redir_url = "http://allmydata.org/TARGET"
2609         def _check2(statuscode, target):
2610             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2611             self.failUnlessReallyEqual(target, redir_url)
2612         d.addCallback(lambda res:
2613                       self.shouldRedirect2("test_POST_DIRURL_check",
2614                                            _check2,
2615                                            self.POST, foo_url,
2616                                            t="check",
2617                                            when_done=redir_url))
2618         d.addCallback(lambda res:
2619                       self.POST(foo_url, t="check", return_to=redir_url))
2620         def _check3(res):
2621             self.failUnless("Healthy :" in res, res)
2622             self.failUnless("Return to file/directory" in res)
2623             self.failUnless(redir_url in res)
2624         d.addCallback(_check3)
2625
2626         d.addCallback(lambda res:
2627                       self.POST(foo_url, t="check", output="JSON"))
2628         def _check_json(res):
2629             data = simplejson.loads(res)
2630             self.failUnless("storage-index" in data)
2631             self.failUnless(data["results"]["healthy"])
2632         d.addCallback(_check_json)
2633
2634         return d
2635
2636     def test_POST_DIRURL_check_and_repair(self):
2637         foo_url = self.public_url + "/foo/"
2638         d = self.POST(foo_url, t="check", repair="true")
2639         def _check(res):
2640             self.failUnless("Healthy :" in res, res)
2641         d.addCallback(_check)
2642         redir_url = "http://allmydata.org/TARGET"
2643         def _check2(statuscode, target):
2644             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2645             self.failUnlessReallyEqual(target, redir_url)
2646         d.addCallback(lambda res:
2647                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2648                                            _check2,
2649                                            self.POST, foo_url,
2650                                            t="check", repair="true",
2651                                            when_done=redir_url))
2652         d.addCallback(lambda res:
2653                       self.POST(foo_url, t="check", return_to=redir_url))
2654         def _check3(res):
2655             self.failUnless("Healthy :" in res)
2656             self.failUnless("Return to file/directory" in res)
2657             self.failUnless(redir_url in res)
2658         d.addCallback(_check3)
2659         return d
2660
2661     def test_POST_FILEURL_mdmf_check(self):
2662         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2663         d = self.POST(quux_url, t="check")
2664         def _check(res):
2665             self.failUnlessIn("Healthy", res)
2666         d.addCallback(_check)
2667         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2668         d.addCallback(lambda ignored:
2669             self.POST(quux_extension_url, t="check"))
2670         d.addCallback(_check)
2671         return d
2672
2673     def test_POST_FILEURL_mdmf_check_and_repair(self):
2674         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2675         d = self.POST(quux_url, t="check", repair="true")
2676         def _check(res):
2677             self.failUnlessIn("Healthy", res)
2678         d.addCallback(_check)
2679         quux_extension_url = "/uri/%s" %\
2680             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2681         d.addCallback(lambda ignored:
2682             self.POST(quux_extension_url, t="check", repair="true"))
2683         d.addCallback(_check)
2684         return d
2685
2686     def wait_for_operation(self, ignored, ophandle):
2687         url = "/operations/" + ophandle
2688         url += "?t=status&output=JSON"
2689         d = self.GET(url)
2690         def _got(res):
2691             data = simplejson.loads(res)
2692             if not data["finished"]:
2693                 d = self.stall(delay=1.0)
2694                 d.addCallback(self.wait_for_operation, ophandle)
2695                 return d
2696             return data
2697         d.addCallback(_got)
2698         return d
2699
2700     def get_operation_results(self, ignored, ophandle, output=None):
2701         url = "/operations/" + ophandle
2702         url += "?t=status"
2703         if output:
2704             url += "&output=" + output
2705         d = self.GET(url)
2706         def _got(res):
2707             if output and output.lower() == "json":
2708                 return simplejson.loads(res)
2709             return res
2710         d.addCallback(_got)
2711         return d
2712
2713     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2714         d = self.shouldFail2(error.Error,
2715                              "test_POST_DIRURL_deepcheck_no_ophandle",
2716                              "400 Bad Request",
2717                              "slow operation requires ophandle=",
2718                              self.POST, self.public_url, t="start-deep-check")
2719         return d
2720
2721     def test_POST_DIRURL_deepcheck(self):
2722         def _check_redirect(statuscode, target):
2723             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2724             self.failUnless(target.endswith("/operations/123"))
2725         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2726                                  self.POST, self.public_url,
2727                                  t="start-deep-check", ophandle="123")
2728         d.addCallback(self.wait_for_operation, "123")
2729         def _check_json(data):
2730             self.failUnlessReallyEqual(data["finished"], True)
2731             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2732             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2733         d.addCallback(_check_json)
2734         d.addCallback(self.get_operation_results, "123", "html")
2735         def _check_html(res):
2736             self.failUnless("Objects Checked: <span>10</span>" in res)
2737             self.failUnless("Objects Healthy: <span>10</span>" in res)
2738         d.addCallback(_check_html)
2739
2740         d.addCallback(lambda res:
2741                       self.GET("/operations/123/"))
2742         d.addCallback(_check_html) # should be the same as without the slash
2743
2744         d.addCallback(lambda res:
2745                       self.shouldFail2(error.Error, "one", "404 Not Found",
2746                                        "No detailed results for SI bogus",
2747                                        self.GET, "/operations/123/bogus"))
2748
2749         foo_si = self._foo_node.get_storage_index()
2750         foo_si_s = base32.b2a(foo_si)
2751         d.addCallback(lambda res:
2752                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2753         def _check_foo_json(res):
2754             data = simplejson.loads(res)
2755             self.failUnlessEqual(data["storage-index"], foo_si_s)
2756             self.failUnless(data["results"]["healthy"])
2757         d.addCallback(_check_foo_json)
2758         return d
2759
2760     def test_POST_DIRURL_deepcheck_and_repair(self):
2761         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2762                       ophandle="124", output="json", followRedirect=True)
2763         d.addCallback(self.wait_for_operation, "124")
2764         def _check_json(data):
2765             self.failUnlessReallyEqual(data["finished"], True)
2766             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2767             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2768             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2769             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2770             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2771             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2772             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2773             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2774             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2775             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2776         d.addCallback(_check_json)
2777         d.addCallback(self.get_operation_results, "124", "html")
2778         def _check_html(res):
2779             self.failUnless("Objects Checked: <span>10</span>" in res)
2780
2781             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2782             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2783             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2784
2785             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2786             self.failUnless("Repairs Successful: <span>0</span>" in res)
2787             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2788
2789             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2790             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2791             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2792         d.addCallback(_check_html)
2793         return d
2794
2795     def test_POST_FILEURL_bad_t(self):
2796         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2797                              "POST to file: bad t=bogus",
2798                              self.POST, self.public_url + "/foo/bar.txt",
2799                              t="bogus")
2800         return d
2801
2802     def test_POST_mkdir(self): # return value?
2803         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2804         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805         d.addCallback(self.failUnlessNodeKeysAre, [])
2806         return d
2807
2808     def test_POST_mkdir_mdmf(self):
2809         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=mdmf")
2810         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2811         d.addCallback(lambda node:
2812             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2813         return d
2814
2815     def test_POST_mkdir_sdmf(self):
2816         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=sdmf")
2817         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2818         d.addCallback(lambda node:
2819             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2820         return d
2821
2822     def test_POST_mkdir_bad_mutable_type(self):
2823         return self.shouldHTTPError("test bad mutable type",
2824                                     400, "Bad Request", "Unknown type: foo",
2825                                     self.POST, self.public_url + \
2826                                     "/foo?t=mkdir&name=newdir&mutable-type=foo")
2827
2828     def test_POST_mkdir_initial_children(self):
2829         (newkids, caps) = self._create_initial_children()
2830         d = self.POST2(self.public_url +
2831                        "/foo?t=mkdir-with-children&name=newdir",
2832                        simplejson.dumps(newkids))
2833         d.addCallback(lambda res:
2834                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2835         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2836         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2837         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2838         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2839         return d
2840
2841     def test_POST_mkdir_initial_children_mdmf(self):
2842         (newkids, caps) = self._create_initial_children()
2843         d = self.POST2(self.public_url +
2844                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=mdmf",
2845                        simplejson.dumps(newkids))
2846         d.addCallback(lambda res:
2847                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2848         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2849         d.addCallback(lambda node:
2850             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2851         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2852         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2853                        caps['filecap1'])
2854         return d
2855
2856     # XXX: Duplication.
2857     def test_POST_mkdir_initial_children_sdmf(self):
2858         (newkids, caps) = self._create_initial_children()
2859         d = self.POST2(self.public_url +
2860                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=sdmf",
2861                        simplejson.dumps(newkids))
2862         d.addCallback(lambda res:
2863                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2864         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2865         d.addCallback(lambda node:
2866             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2867         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2868         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2869                        caps['filecap1'])
2870         return d
2871
2872     def test_POST_mkdir_initial_children_bad_mutable_type(self):
2873         (newkids, caps) = self._create_initial_children()
2874         return self.shouldHTTPError("test bad mutable type",
2875                                     400, "Bad Request", "Unknown type: foo",
2876                                     self.POST, self.public_url + \
2877                                     "/foo?t=mkdir-with-children&name=newdir&mutable-type=foo",
2878                                     simplejson.dumps(newkids))
2879
2880     def test_POST_mkdir_immutable(self):
2881         (newkids, caps) = self._create_immutable_children()
2882         d = self.POST2(self.public_url +
2883                        "/foo?t=mkdir-immutable&name=newdir",
2884                        simplejson.dumps(newkids))
2885         d.addCallback(lambda res:
2886                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2888         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2889         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2890         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2891         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2892         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2893         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2894         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2895         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2896         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2897         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2898         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2899         return d
2900
2901     def test_POST_mkdir_immutable_bad(self):
2902         (newkids, caps) = self._create_initial_children()
2903         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2904                              "400 Bad Request",
2905                              "needed to be immutable but was not",
2906                              self.POST2,
2907                              self.public_url +
2908                              "/foo?t=mkdir-immutable&name=newdir",
2909                              simplejson.dumps(newkids))
2910         return d
2911
2912     def test_POST_mkdir_2(self):
2913         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2914         d.addCallback(lambda res:
2915                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2916         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917         d.addCallback(self.failUnlessNodeKeysAre, [])
2918         return d
2919
2920     def test_POST_mkdirs_2(self):
2921         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2922         d.addCallback(lambda res:
2923                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2924         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2925         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2926         d.addCallback(self.failUnlessNodeKeysAre, [])
2927         return d
2928
2929     def test_POST_mkdir_no_parentdir_noredirect(self):
2930         d = self.POST("/uri?t=mkdir")
2931         def _after_mkdir(res):
2932             uri.DirectoryURI.init_from_string(res)
2933         d.addCallback(_after_mkdir)
2934         return d
2935
2936     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2937         d = self.POST("/uri?t=mkdir&mutable-type=mdmf")
2938         def _after_mkdir(res):
2939             u = uri.from_string(res)
2940             # Check that this is an MDMF writecap
2941             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2942         d.addCallback(_after_mkdir)
2943         return d
2944
2945     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2946         d = self.POST("/uri?t=mkdir&mutable-type=sdmf")
2947         def _after_mkdir(res):
2948             u = uri.from_string(res)
2949             self.failUnlessIsInstance(u, uri.DirectoryURI)
2950         d.addCallback(_after_mkdir)
2951         return d
2952
2953     def test_POST_mkdir_no_parentdir_noredirect_bad_mutable_type(self):
2954         return self.shouldHTTPError("test bad mutable type",
2955                                     400, "Bad Request", "Unknown type: foo",
2956                                     self.POST, self.public_url + \
2957                                     "/uri?t=mkdir&mutable-type=foo")
2958
2959     def test_POST_mkdir_no_parentdir_noredirect2(self):
2960         # make sure form-based arguments (as on the welcome page) still work
2961         d = self.POST("/uri", t="mkdir")
2962         def _after_mkdir(res):
2963             uri.DirectoryURI.init_from_string(res)
2964         d.addCallback(_after_mkdir)
2965         d.addErrback(self.explain_web_error)
2966         return d
2967
2968     def test_POST_mkdir_no_parentdir_redirect(self):
2969         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2970         d.addBoth(self.shouldRedirect, None, statuscode='303')
2971         def _check_target(target):
2972             target = urllib.unquote(target)
2973             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2974         d.addCallback(_check_target)
2975         return d
2976
2977     def test_POST_mkdir_no_parentdir_redirect2(self):
2978         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2979         d.addBoth(self.shouldRedirect, None, statuscode='303')
2980         def _check_target(target):
2981             target = urllib.unquote(target)
2982             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2983         d.addCallback(_check_target)
2984         d.addErrback(self.explain_web_error)
2985         return d
2986
2987     def _make_readonly(self, u):
2988         ro_uri = uri.from_string(u).get_readonly()
2989         if ro_uri is None:
2990             return None
2991         return ro_uri.to_string()
2992
2993     def _create_initial_children(self):
2994         contents, n, filecap1 = self.makefile(12)
2995         md1 = {"metakey1": "metavalue1"}
2996         filecap2 = make_mutable_file_uri()
2997         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2998         filecap3 = node3.get_readonly_uri()
2999         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3000         dircap = DirectoryNode(node4, None, None).get_uri()
3001         mdmfcap = make_mutable_file_uri(mdmf=True)
3002         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3003         emptydircap = "URI:DIR2-LIT:"
3004         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3005                                                       "ro_uri": self._make_readonly(filecap1),
3006                                                       "metadata": md1, }],
3007                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3008                                                       "ro_uri": self._make_readonly(filecap2)}],
3009                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3010                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3011                                                       "ro_uri": unknown_rocap}],
3012                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3013                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3014                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3015                                                       "ro_uri": self._make_readonly(dircap)}],
3016                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3017                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3018                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3019                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3020                    }
3021         return newkids, {'filecap1': filecap1,
3022                          'filecap2': filecap2,
3023                          'filecap3': filecap3,
3024                          'unknown_rwcap': unknown_rwcap,
3025                          'unknown_rocap': unknown_rocap,
3026                          'unknown_immcap': unknown_immcap,
3027                          'dircap': dircap,
3028                          'litdircap': litdircap,
3029                          'emptydircap': emptydircap,
3030                          'mdmfcap': mdmfcap}
3031
3032     def _create_immutable_children(self):
3033         contents, n, filecap1 = self.makefile(12)
3034         md1 = {"metakey1": "metavalue1"}
3035         tnode = create_chk_filenode("immutable directory contents\n"*10)
3036         dnode = DirectoryNode(tnode, None, None)
3037         assert not dnode.is_mutable()
3038         immdircap = dnode.get_uri()
3039         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3040         emptydircap = "URI:DIR2-LIT:"
3041         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3042                                                       "metadata": md1, }],
3043                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3044                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3045                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3046                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3047                    }
3048         return newkids, {'filecap1': filecap1,
3049                          'unknown_immcap': unknown_immcap,
3050                          'immdircap': immdircap,
3051                          'litdircap': litdircap,
3052                          'emptydircap': emptydircap}
3053
3054     def test_POST_mkdir_no_parentdir_initial_children(self):
3055         (newkids, caps) = self._create_initial_children()
3056         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3057         def _after_mkdir(res):
3058             self.failUnless(res.startswith("URI:DIR"), res)
3059             n = self.s.create_node_from_uri(res)
3060             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3061             d2.addCallback(lambda ign:
3062                            self.failUnlessROChildURIIs(n, u"child-imm",
3063                                                        caps['filecap1']))
3064             d2.addCallback(lambda ign:
3065                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3066                                                        caps['filecap2']))
3067             d2.addCallback(lambda ign:
3068                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3069                                                        caps['filecap3']))
3070             d2.addCallback(lambda ign:
3071                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3072                                                        caps['unknown_rwcap']))
3073             d2.addCallback(lambda ign:
3074                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3075                                                        caps['unknown_rocap']))
3076             d2.addCallback(lambda ign:
3077                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3078                                                        caps['unknown_immcap']))
3079             d2.addCallback(lambda ign:
3080                            self.failUnlessRWChildURIIs(n, u"dirchild",
3081                                                        caps['dircap']))
3082             return d2
3083         d.addCallback(_after_mkdir)
3084         return d
3085
3086     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3087         # the regular /uri?t=mkdir operation is specified to ignore its body.
3088         # Only t=mkdir-with-children pays attention to it.
3089         (newkids, caps) = self._create_initial_children()
3090         d = self.shouldHTTPError("POST t=mkdir unexpected children",
3091                                  400, "Bad Request",
3092                                  "t=mkdir does not accept children=, "
3093                                  "try t=mkdir-with-children instead",
3094                                  self.POST2, "/uri?t=mkdir", # without children
3095                                  simplejson.dumps(newkids))
3096         return d
3097
3098     def test_POST_noparent_bad(self):
3099         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
3100                                  "/uri accepts only PUT, PUT?t=mkdir, "
3101                                  "POST?t=upload, and POST?t=mkdir",
3102                                  self.POST, "/uri?t=bogus")
3103         return d
3104
3105     def test_POST_mkdir_no_parentdir_immutable(self):
3106         (newkids, caps) = self._create_immutable_children()
3107         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3108         def _after_mkdir(res):
3109             self.failUnless(res.startswith("URI:DIR"), res)
3110             n = self.s.create_node_from_uri(res)
3111             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3112             d2.addCallback(lambda ign:
3113                            self.failUnlessROChildURIIs(n, u"child-imm",
3114                                                           caps['filecap1']))
3115             d2.addCallback(lambda ign:
3116                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3117                                                           caps['unknown_immcap']))
3118             d2.addCallback(lambda ign:
3119                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3120                                                           caps['immdircap']))
3121             d2.addCallback(lambda ign:
3122                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3123                                                           caps['litdircap']))
3124             d2.addCallback(lambda ign:
3125                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3126                                                           caps['emptydircap']))
3127             return d2
3128         d.addCallback(_after_mkdir)
3129         return d
3130
3131     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3132         (newkids, caps) = self._create_initial_children()
3133         d = self.shouldFail2(error.Error,
3134                              "test_POST_mkdir_no_parentdir_immutable_bad",
3135                              "400 Bad Request",
3136                              "needed to be immutable but was not",
3137                              self.POST2,
3138                              "/uri?t=mkdir-immutable",
3139                              simplejson.dumps(newkids))
3140         return d
3141
3142     def test_welcome_page_mkdir_button(self):
3143         # Fetch the welcome page.
3144         d = self.GET("/")
3145         def _after_get_welcome_page(res):
3146             MKDIR_BUTTON_RE = re.compile(
3147                 '<form action="([^"]*)" method="post".*?'
3148                 '<input type="hidden" name="t" value="([^"]*)" />'
3149                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3150                 '<input type="submit" value="Create a directory" />',
3151                 re.I)
3152             mo = MKDIR_BUTTON_RE.search(res)
3153             formaction = mo.group(1)
3154             formt = mo.group(2)
3155             formaname = mo.group(3)
3156             formavalue = mo.group(4)
3157             return (formaction, formt, formaname, formavalue)
3158         d.addCallback(_after_get_welcome_page)
3159         def _after_parse_form(res):
3160             (formaction, formt, formaname, formavalue) = res
3161             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3162         d.addCallback(_after_parse_form)
3163         d.addBoth(self.shouldRedirect, None, statuscode='303')
3164         return d
3165
3166     def test_POST_mkdir_replace(self): # return value?
3167         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3168         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3169         d.addCallback(self.failUnlessNodeKeysAre, [])
3170         return d
3171
3172     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3173         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3174         d.addBoth(self.shouldFail, error.Error,
3175                   "POST_mkdir_no_replace_queryarg",
3176                   "409 Conflict",
3177                   "There was already a child by that name, and you asked me "
3178                   "to not replace it")
3179         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3180         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3181         return d
3182
3183     def test_POST_mkdir_no_replace_field(self): # return value?
3184         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3185                       replace="false")
3186         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3187                   "409 Conflict",
3188                   "There was already a child by that name, and you asked me "
3189                   "to not replace it")
3190         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3191         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3192         return d
3193
3194     def test_POST_mkdir_whendone_field(self):
3195         d = self.POST(self.public_url + "/foo",
3196                       t="mkdir", name="newdir", when_done="/THERE")
3197         d.addBoth(self.shouldRedirect, "/THERE")
3198         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3199         d.addCallback(self.failUnlessNodeKeysAre, [])
3200         return d
3201
3202     def test_POST_mkdir_whendone_queryarg(self):
3203         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3204                       t="mkdir", name="newdir")
3205         d.addBoth(self.shouldRedirect, "/THERE")
3206         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3207         d.addCallback(self.failUnlessNodeKeysAre, [])
3208         return d
3209
3210     def test_POST_bad_t(self):
3211         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
3212                              "POST to a directory with bad t=BOGUS",
3213                              self.POST, self.public_url + "/foo", t="BOGUS")
3214         return d
3215
3216     def test_POST_set_children(self, command_name="set_children"):
3217         contents9, n9, newuri9 = self.makefile(9)
3218         contents10, n10, newuri10 = self.makefile(10)
3219         contents11, n11, newuri11 = self.makefile(11)
3220
3221         reqbody = """{
3222                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3223                                                 "size": 0,
3224                                                 "metadata": {
3225                                                   "ctime": 1002777696.7564139,
3226                                                   "mtime": 1002777696.7564139
3227                                                  }
3228                                                } ],
3229                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3230                                                 "size": 1,
3231                                                 "metadata": {
3232                                                   "ctime": 1002777696.7564139,
3233                                                   "mtime": 1002777696.7564139
3234                                                  }
3235                                                } ],
3236                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3237                                                 "size": 2,
3238                                                 "metadata": {
3239                                                   "ctime": 1002777696.7564139,
3240                                                   "mtime": 1002777696.7564139
3241                                                  }
3242                                                } ]
3243                     }""" % (newuri9, newuri10, newuri11)
3244
3245         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3246
3247         d = client.getPage(url, method="POST", postdata=reqbody)
3248         def _then(res):
3249             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3250             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3251             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3252
3253         d.addCallback(_then)
3254         d.addErrback(self.dump_error)
3255         return d
3256
3257     def test_POST_set_children_with_hyphen(self):
3258         return self.test_POST_set_children(command_name="set-children")
3259
3260     def test_POST_link_uri(self):
3261         contents, n, newuri = self.makefile(8)
3262         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3263         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3264         d.addCallback(lambda res:
3265                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3266                                                       contents))
3267         return d
3268
3269     def test_POST_link_uri_replace(self):
3270         contents, n, newuri = self.makefile(8)
3271         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3272         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3273         d.addCallback(lambda res:
3274                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3275                                                       contents))
3276         return d
3277
3278     def test_POST_link_uri_unknown_bad(self):
3279         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3280         d.addBoth(self.shouldFail, error.Error,
3281                   "POST_link_uri_unknown_bad",
3282                   "400 Bad Request",
3283                   "unknown cap in a write slot")
3284         return d
3285
3286     def test_POST_link_uri_unknown_ro_good(self):
3287         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3288         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3289         return d
3290
3291     def test_POST_link_uri_unknown_imm_good(self):
3292         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3293         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3294         return d
3295
3296     def test_POST_link_uri_no_replace_queryarg(self):
3297         contents, n, newuri = self.makefile(8)
3298         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3299                       name="bar.txt", uri=newuri)
3300         d.addBoth(self.shouldFail, error.Error,
3301                   "POST_link_uri_no_replace_queryarg",
3302                   "409 Conflict",
3303                   "There was already a child by that name, and you asked me "
3304                   "to not replace it")
3305         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3306         d.addCallback(self.failUnlessIsBarDotTxt)
3307         return d
3308
3309     def test_POST_link_uri_no_replace_field(self):
3310         contents, n, newuri = self.makefile(8)
3311         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3312                       name="bar.txt", uri=newuri)
3313         d.addBoth(self.shouldFail, error.Error,
3314                   "POST_link_uri_no_replace_field",
3315                   "409 Conflict",
3316                   "There was already a child by that name, and you asked me "
3317                   "to not replace it")
3318         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3319         d.addCallback(self.failUnlessIsBarDotTxt)
3320         return d
3321
3322     def test_POST_delete(self, command_name='delete'):
3323         d = self._foo_node.list()
3324         def _check_before(children):
3325             self.failUnless(u"bar.txt" in children)
3326         d.addCallback(_check_before)
3327         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3328         d.addCallback(lambda res: self._foo_node.list())
3329         def _check_after(children):
3330             self.failIf(u"bar.txt" in children)
3331         d.addCallback(_check_after)
3332         return d
3333
3334     def test_POST_unlink(self):
3335         return self.test_POST_delete(command_name='unlink')
3336
3337     def test_POST_rename_file(self):
3338         d = self.POST(self.public_url + "/foo", t="rename",
3339                       from_name="bar.txt", to_name='wibble.txt')
3340         d.addCallback(lambda res:
3341                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3342         d.addCallback(lambda res:
3343                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3344         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3345         d.addCallback(self.failUnlessIsBarDotTxt)
3346         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3347         d.addCallback(self.failUnlessIsBarJSON)
3348         return d
3349
3350     def test_POST_rename_file_redundant(self):
3351         d = self.POST(self.public_url + "/foo", t="rename",
3352                       from_name="bar.txt", to_name='bar.txt')
3353         d.addCallback(lambda res:
3354                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3355         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3356         d.addCallback(self.failUnlessIsBarDotTxt)
3357         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3358         d.addCallback(self.failUnlessIsBarJSON)
3359         return d
3360
3361     def test_POST_rename_file_replace(self):
3362         # rename a file and replace a directory with it
3363         d = self.POST(self.public_url + "/foo", t="rename",
3364                       from_name="bar.txt", to_name='empty')
3365         d.addCallback(lambda res:
3366                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3367         d.addCallback(lambda res:
3368                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3370         d.addCallback(self.failUnlessIsBarDotTxt)
3371         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3372         d.addCallback(self.failUnlessIsBarJSON)
3373         return d
3374
3375     def test_POST_rename_file_no_replace_queryarg(self):
3376         # rename a file and replace a directory with it
3377         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3378                       from_name="bar.txt", to_name='empty')
3379         d.addBoth(self.shouldFail, error.Error,
3380                   "POST_rename_file_no_replace_queryarg",
3381                   "409 Conflict",
3382                   "There was already a child by that name, and you asked me "
3383                   "to not replace it")
3384         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3385         d.addCallback(self.failUnlessIsEmptyJSON)
3386         return d
3387
3388     def test_POST_rename_file_no_replace_field(self):
3389         # rename a file and replace a directory with it
3390         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3391                       from_name="bar.txt", to_name='empty')
3392         d.addBoth(self.shouldFail, error.Error,
3393                   "POST_rename_file_no_replace_field",
3394                   "409 Conflict",
3395                   "There was already a child by that name, and you asked me "
3396                   "to not replace it")
3397         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3398         d.addCallback(self.failUnlessIsEmptyJSON)
3399         return d
3400
3401     def failUnlessIsEmptyJSON(self, res):
3402         data = simplejson.loads(res)
3403         self.failUnlessEqual(data[0], "dirnode", data)
3404         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3405
3406     def test_POST_rename_file_slash_fail(self):
3407         d = self.POST(self.public_url + "/foo", t="rename",
3408                       from_name="bar.txt", to_name='kirk/spock.txt')
3409         d.addBoth(self.shouldFail, error.Error,
3410                   "test_POST_rename_file_slash_fail",
3411                   "400 Bad Request",
3412                   "to_name= may not contain a slash",
3413                   )
3414         d.addCallback(lambda res:
3415                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3416         return d
3417
3418     def test_POST_rename_dir(self):
3419         d = self.POST(self.public_url, t="rename",
3420                       from_name="foo", to_name='plunk')
3421         d.addCallback(lambda res:
3422                       self.failIfNodeHasChild(self.public_root, u"foo"))
3423         d.addCallback(lambda res:
3424                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3425         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3426         d.addCallback(self.failUnlessIsFooJSON)
3427         return d
3428
3429     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3430         """ If target is not None then the redirection has to go to target.  If
3431         statuscode is not None then the redirection has to be accomplished with
3432         that HTTP status code."""
3433         if not isinstance(res, failure.Failure):
3434             to_where = (target is None) and "somewhere" or ("to " + target)
3435             self.fail("%s: we were expecting to get redirected %s, not get an"
3436                       " actual page: %s" % (which, to_where, res))
3437         res.trap(error.PageRedirect)
3438         if statuscode is not None:
3439             self.failUnlessReallyEqual(res.value.status, statuscode,
3440                                        "%s: not a redirect" % which)
3441         if target is not None:
3442             # the PageRedirect does not seem to capture the uri= query arg
3443             # properly, so we can't check for it.
3444             realtarget = self.webish_url + target
3445             self.failUnlessReallyEqual(res.value.location, realtarget,
3446                                        "%s: wrong target" % which)
3447         return res.value.location
3448
3449     def test_GET_URI_form(self):
3450         base = "/uri?uri=%s" % self._bar_txt_uri
3451         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3452         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3453         d = self.GET(base)
3454         d.addBoth(self.shouldRedirect, targetbase)
3455         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3456         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3457         d.addCallback(lambda res: self.GET(base+"&t=json"))
3458         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3459         d.addCallback(self.log, "about to get file by uri")
3460         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3461         d.addCallback(self.failUnlessIsBarDotTxt)
3462         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3463         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3464                                            followRedirect=True))
3465         d.addCallback(self.failUnlessIsFooJSON)
3466         d.addCallback(self.log, "got dir by uri")
3467
3468         return d
3469
3470     def test_GET_URI_form_bad(self):
3471         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3472                              "400 Bad Request", "GET /uri requires uri=",
3473                              self.GET, "/uri")
3474         return d
3475
3476     def test_GET_rename_form(self):
3477         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3478                      followRedirect=True)
3479         def _check(res):
3480             self.failUnless('name="when_done" value="."' in res, res)
3481             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3482         d.addCallback(_check)
3483         return d
3484
3485     def log(self, res, msg):
3486         #print "MSG: %s  RES: %s" % (msg, res)
3487         log.msg(msg)
3488         return res
3489
3490     def test_GET_URI_URL(self):
3491         base = "/uri/%s" % self._bar_txt_uri
3492         d = self.GET(base)
3493         d.addCallback(self.failUnlessIsBarDotTxt)
3494         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3495         d.addCallback(self.failUnlessIsBarDotTxt)
3496         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3497         d.addCallback(self.failUnlessIsBarDotTxt)
3498         return d
3499
3500     def test_GET_URI_URL_dir(self):
3501         base = "/uri/%s?t=json" % self._foo_uri
3502         d = self.GET(base)
3503         d.addCallback(self.failUnlessIsFooJSON)
3504         return d
3505
3506     def test_GET_URI_URL_missing(self):
3507         base = "/uri/%s" % self._bad_file_uri
3508         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3509                                  http.GONE, None, "NotEnoughSharesError",
3510                                  self.GET, base)
3511         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3512         # here? we must arrange for a download to fail after target.open()
3513         # has been called, and then inspect the response to see that it is
3514         # shorter than we expected.
3515         return d
3516
3517     def test_PUT_DIRURL_uri(self):
3518         d = self.s.create_dirnode()
3519         def _made_dir(dn):
3520             new_uri = dn.get_uri()
3521             # replace /foo with a new (empty) directory
3522             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3523             d.addCallback(lambda res:
3524                           self.failUnlessReallyEqual(res.strip(), new_uri))
3525             d.addCallback(lambda res:
3526                           self.failUnlessRWChildURIIs(self.public_root,
3527                                                       u"foo",
3528                                                       new_uri))
3529             return d
3530         d.addCallback(_made_dir)
3531         return d
3532
3533     def test_PUT_DIRURL_uri_noreplace(self):
3534         d = self.s.create_dirnode()
3535         def _made_dir(dn):
3536             new_uri = dn.get_uri()
3537             # replace /foo with a new (empty) directory, but ask that
3538             # replace=false, so it should fail
3539             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3540                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3541                                  self.PUT,
3542                                  self.public_url + "/foo?t=uri&replace=false",
3543                                  new_uri)
3544             d.addCallback(lambda res:
3545                           self.failUnlessRWChildURIIs(self.public_root,
3546                                                       u"foo",
3547                                                       self._foo_uri))
3548             return d
3549         d.addCallback(_made_dir)
3550         return d
3551
3552     def test_PUT_DIRURL_bad_t(self):
3553         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3554                                  "400 Bad Request", "PUT to a directory",
3555                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
3556         d.addCallback(lambda res:
3557                       self.failUnlessRWChildURIIs(self.public_root,
3558                                                   u"foo",
3559                                                   self._foo_uri))
3560         return d
3561
3562     def test_PUT_NEWFILEURL_uri(self):
3563         contents, n, new_uri = self.makefile(8)
3564         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3565         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3566         d.addCallback(lambda res:
3567                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3568                                                       contents))
3569         return d
3570
3571     def test_PUT_NEWFILEURL_mdmf(self):
3572         new_contents = self.NEWFILE_CONTENTS * 300000
3573         d = self.PUT(self.public_url + \
3574                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
3575                      new_contents)
3576         d.addCallback(lambda ignored:
3577             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3578         def _got_json(json):
3579             data = simplejson.loads(json)
3580             data = data[1]
3581             self.failUnlessIn("mutable-type", data)
3582             self.failUnlessEqual(data['mutable-type'], "mdmf")
3583             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3584             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3585         d.addCallback(_got_json)
3586         return d
3587
3588     def test_PUT_NEWFILEURL_sdmf(self):
3589         new_contents = self.NEWFILE_CONTENTS * 300000
3590         d = self.PUT(self.public_url + \
3591                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
3592                      new_contents)
3593         d.addCallback(lambda ignored:
3594             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3595         def _got_json(json):
3596             data = simplejson.loads(json)
3597             data = data[1]
3598             self.failUnlessIn("mutable-type", data)
3599             self.failUnlessEqual(data['mutable-type'], "sdmf")
3600         d.addCallback(_got_json)
3601         return d
3602
3603     def test_PUT_NEWFILEURL_bad_mutable_type(self):
3604        new_contents = self.NEWFILE_CONTENTS * 300000
3605        return self.shouldHTTPError("test bad mutable type",
3606                                    400, "Bad Request", "Unknown type: foo",
3607                                    self.PUT, self.public_url + \
3608                                    "/foo/foo.txt?mutable=true&mutable-type=foo",
3609                                    new_contents)
3610
3611     def test_PUT_NEWFILEURL_uri_replace(self):
3612         contents, n, new_uri = self.makefile(8)
3613         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3614         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3615         d.addCallback(lambda res:
3616                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3617                                                       contents))
3618         return d
3619
3620     def test_PUT_NEWFILEURL_uri_no_replace(self):
3621         contents, n, new_uri = self.makefile(8)
3622         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3623         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
3624                   "409 Conflict",
3625                   "There was already a child by that name, and you asked me "
3626                   "to not replace it")
3627         return d
3628
3629     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3630         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3631         d.addBoth(self.shouldFail, error.Error,
3632                   "POST_put_uri_unknown_bad",
3633                   "400 Bad Request",
3634                   "unknown cap in a write slot")
3635         return d
3636
3637     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3638         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3639         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3640                       u"put-future-ro.txt")
3641         return d
3642
3643     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3644         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3645         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3646                       u"put-future-imm.txt")
3647         return d
3648
3649     def test_PUT_NEWFILE_URI(self):
3650         file_contents = "New file contents here\n"
3651         d = self.PUT("/uri", file_contents)
3652         def _check(uri):
3653             assert isinstance(uri, str), uri
3654             self.failUnless(uri in FakeCHKFileNode.all_contents)
3655             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3656                                        file_contents)
3657             return self.GET("/uri/%s" % uri)
3658         d.addCallback(_check)
3659         def _check2(res):
3660             self.failUnlessReallyEqual(res, file_contents)
3661         d.addCallback(_check2)
3662         return d
3663
3664     def test_PUT_NEWFILE_URI_not_mutable(self):
3665         file_contents = "New file contents here\n"
3666         d = self.PUT("/uri?mutable=false", file_contents)
3667         def _check(uri):
3668             assert isinstance(uri, str), uri
3669             self.failUnless(uri in FakeCHKFileNode.all_contents)
3670             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3671                                        file_contents)
3672             return self.GET("/uri/%s" % uri)
3673         d.addCallback(_check)
3674         def _check2(res):
3675             self.failUnlessReallyEqual(res, file_contents)
3676         d.addCallback(_check2)
3677         return d
3678
3679     def test_PUT_NEWFILE_URI_only_PUT(self):
3680         d = self.PUT("/uri?t=bogus", "")
3681         d.addBoth(self.shouldFail, error.Error,
3682                   "PUT_NEWFILE_URI_only_PUT",
3683                   "400 Bad Request",
3684                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3685         return d
3686
3687     def test_PUT_NEWFILE_URI_mutable(self):
3688         file_contents = "New file contents here\n"
3689         d = self.PUT("/uri?mutable=true", file_contents)
3690         def _check1(filecap):
3691             filecap = filecap.strip()
3692             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3693             self.filecap = filecap
3694             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3695             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3696             n = self.s.create_node_from_uri(filecap)
3697             return n.download_best_version()
3698         d.addCallback(_check1)
3699         def _check2(data):
3700             self.failUnlessReallyEqual(data, file_contents)
3701             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3702         d.addCallback(_check2)
3703         def _check3(res):
3704             self.failUnlessReallyEqual(res, file_contents)
3705         d.addCallback(_check3)
3706         return d
3707
3708     def test_PUT_mkdir(self):
3709         d = self.PUT("/uri?t=mkdir", "")
3710         def _check(uri):
3711             n = self.s.create_node_from_uri(uri.strip())
3712             d2 = self.failUnlessNodeKeysAre(n, [])
3713             d2.addCallback(lambda res:
3714                            self.GET("/uri/%s?t=json" % uri))
3715             return d2
3716         d.addCallback(_check)
3717         d.addCallback(self.failUnlessIsEmptyJSON)
3718         return d
3719
3720     def test_PUT_mkdir_mdmf(self):
3721         d = self.PUT("/uri?t=mkdir&mutable-type=mdmf", "")
3722         def _got(res):
3723             u = uri.from_string(res)
3724             # Check that this is an MDMF writecap
3725             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3726         d.addCallback(_got)
3727         return d
3728
3729     def test_PUT_mkdir_sdmf(self):
3730         d = self.PUT("/uri?t=mkdir&mutable-type=sdmf", "")
3731         def _got(res):
3732             u = uri.from_string(res)
3733             self.failUnlessIsInstance(u, uri.DirectoryURI)
3734         d.addCallback(_got)
3735         return d
3736
3737     def test_PUT_mkdir_bad_mutable_type(self):
3738         return self.shouldHTTPError("bad mutable type",
3739                                     400, "Bad Request", "Unknown type: foo",
3740                                     self.PUT, "/uri?t=mkdir&mutable-type=foo",
3741                                     "")
3742
3743     def test_POST_check(self):
3744         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3745         def _done(res):
3746             # this returns a string form of the results, which are probably
3747             # None since we're using fake filenodes.
3748             # TODO: verify that the check actually happened, by changing
3749             # FakeCHKFileNode to count how many times .check() has been
3750             # called.
3751             pass
3752         d.addCallback(_done)
3753         return d
3754
3755
3756     def test_PUT_update_at_offset(self):
3757         file_contents = "test file" * 100000 # about 900 KiB
3758         d = self.PUT("/uri?mutable=true", file_contents)
3759         def _then(filecap):
3760             self.filecap = filecap
3761             new_data = file_contents[:100]
3762             new = "replaced and so on"
3763             new_data += new
3764             new_data += file_contents[len(new_data):]
3765             assert len(new_data) == len(file_contents)
3766             self.new_data = new_data
3767         d.addCallback(_then)
3768         d.addCallback(lambda ignored:
3769             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3770                      "replaced and so on"))
3771         def _get_data(filecap):
3772             n = self.s.create_node_from_uri(filecap)
3773             return n.download_best_version()
3774         d.addCallback(_get_data)
3775         d.addCallback(lambda results:
3776             self.failUnlessEqual(results, self.new_data))
3777         # Now try appending things to the file
3778         d.addCallback(lambda ignored:
3779             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3780                      "puppies" * 100))
3781         d.addCallback(_get_data)
3782         d.addCallback(lambda results:
3783             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3784         # and try replacing the beginning of the file
3785         d.addCallback(lambda ignored:
3786             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3787         d.addCallback(_get_data)
3788         d.addCallback(lambda results:
3789             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3790         return d
3791
3792     def test_PUT_update_at_invalid_offset(self):
3793         file_contents = "test file" * 100000 # about 900 KiB
3794         d = self.PUT("/uri?mutable=true", file_contents)
3795         def _then(filecap):
3796             self.filecap = filecap
3797         d.addCallback(_then)
3798         # Negative offsets should cause an error.
3799         d.addCallback(lambda ignored:
3800             self.shouldHTTPError("test mutable invalid offset negative",
3801                                  400, "Bad Request",
3802                                  "Invalid offset",
3803                                  self.PUT,
3804                                  "/uri/%s?offset=-1" % self.filecap,
3805                                  "foo"))
3806         return d
3807
3808     def test_PUT_update_at_offset_immutable(self):
3809         file_contents = "Test file" * 100000
3810         d = self.PUT("/uri", file_contents)
3811         def _then(filecap):
3812             self.filecap = filecap
3813         d.addCallback(_then)
3814         d.addCallback(lambda ignored:
3815             self.shouldHTTPError("test immutable update",
3816                                  400, "Bad Request",
3817                                  "immutable",
3818                                  self.PUT,
3819                                  "/uri/%s?offset=50" % self.filecap,
3820                                  "foo"))
3821         return d
3822
3823
3824     def test_bad_method(self):
3825         url = self.webish_url + self.public_url + "/foo/bar.txt"
3826         d = self.shouldHTTPError("test_bad_method",
3827                                  501, "Not Implemented",
3828                                  "I don't know how to treat a BOGUS request.",
3829                                  client.getPage, url, method="BOGUS")
3830         return d
3831
3832     def test_short_url(self):
3833         url = self.webish_url + "/uri"
3834         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3835                                  "I don't know how to treat a DELETE request.",
3836                                  client.getPage, url, method="DELETE")
3837         return d
3838
3839     def test_ophandle_bad(self):
3840         url = self.webish_url + "/operations/bogus?t=status"
3841         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3842                                  "unknown/expired handle 'bogus'",
3843                                  client.getPage, url)
3844         return d
3845
3846     def test_ophandle_cancel(self):
3847         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3848                       followRedirect=True)
3849         d.addCallback(lambda ignored:
3850                       self.GET("/operations/128?t=status&output=JSON"))
3851         def _check1(res):
3852             data = simplejson.loads(res)
3853             self.failUnless("finished" in data, res)
3854             monitor = self.ws.root.child_operations.handles["128"][0]
3855             d = self.POST("/operations/128?t=cancel&output=JSON")
3856             def _check2(res):
3857                 data = simplejson.loads(res)
3858                 self.failUnless("finished" in data, res)
3859                 # t=cancel causes the handle to be forgotten
3860                 self.failUnless(monitor.is_cancelled())
3861             d.addCallback(_check2)
3862             return d
3863         d.addCallback(_check1)
3864         d.addCallback(lambda ignored:
3865                       self.shouldHTTPError("test_ophandle_cancel",
3866                                            404, "404 Not Found",
3867                                            "unknown/expired handle '128'",
3868                                            self.GET,
3869                                            "/operations/128?t=status&output=JSON"))
3870         return d
3871
3872     def test_ophandle_retainfor(self):
3873         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3874                       followRedirect=True)
3875         d.addCallback(lambda ignored:
3876                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3877         def _check1(res):
3878             data = simplejson.loads(res)
3879             self.failUnless("finished" in data, res)
3880         d.addCallback(_check1)
3881         # the retain-for=0 will cause the handle to be expired very soon
3882         d.addCallback(lambda ign:
3883             self.clock.advance(2.0))
3884         d.addCallback(lambda ignored:
3885                       self.shouldHTTPError("test_ophandle_retainfor",
3886                                            404, "404 Not Found",
3887                                            "unknown/expired handle '129'",
3888                                            self.GET,
3889                                            "/operations/129?t=status&output=JSON"))
3890         return d
3891
3892     def test_ophandle_release_after_complete(self):
3893         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3894                       followRedirect=True)
3895         d.addCallback(self.wait_for_operation, "130")
3896         d.addCallback(lambda ignored:
3897                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3898         # the release-after-complete=true will cause the handle to be expired
3899         d.addCallback(lambda ignored:
3900                       self.shouldHTTPError("test_ophandle_release_after_complete",
3901                                            404, "404 Not Found",
3902                                            "unknown/expired handle '130'",
3903                                            self.GET,
3904                                            "/operations/130?t=status&output=JSON"))
3905         return d
3906
3907     def test_uncollected_ophandle_expiration(self):
3908         # uncollected ophandles should expire after 4 days
3909         def _make_uncollected_ophandle(ophandle):
3910             d = self.POST(self.public_url +
3911                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3912                           followRedirect=False)
3913             # When we start the operation, the webapi server will want
3914             # to redirect us to the page for the ophandle, so we get
3915             # confirmation that the operation has started. If the
3916             # manifest operation has finished by the time we get there,
3917             # following that redirect (by setting followRedirect=True
3918             # above) has the side effect of collecting the ophandle that
3919             # we've just created, which means that we can't use the
3920             # ophandle to test the uncollected timeout anymore. So,
3921             # instead, catch the 302 here and don't follow it.
3922             d.addBoth(self.should302, "uncollected_ophandle_creation")
3923             return d
3924         # Create an ophandle, don't collect it, then advance the clock by
3925         # 4 days - 1 second and make sure that the ophandle is still there.
3926         d = _make_uncollected_ophandle(131)
3927         d.addCallback(lambda ign:
3928             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3929         d.addCallback(lambda ign:
3930             self.GET("/operations/131?t=status&output=JSON"))
3931         def _check1(res):
3932             data = simplejson.loads(res)
3933             self.failUnless("finished" in data, res)
3934         d.addCallback(_check1)
3935         # Create an ophandle, don't collect it, then try to collect it
3936         # after 4 days. It should be gone.
3937         d.addCallback(lambda ign:
3938             _make_uncollected_ophandle(132))
3939         d.addCallback(lambda ign:
3940             self.clock.advance(96*60*60))
3941         d.addCallback(lambda ign:
3942             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3943                                  404, "404 Not Found",
3944                                  "unknown/expired handle '132'",
3945                                  self.GET,
3946                                  "/operations/132?t=status&output=JSON"))
3947         return d
3948
3949     def test_collected_ophandle_expiration(self):
3950         # collected ophandles should expire after 1 day
3951         def _make_collected_ophandle(ophandle):
3952             d = self.POST(self.public_url +
3953                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3954                           followRedirect=True)
3955             # By following the initial redirect, we collect the ophandle
3956             # we've just created.
3957             return d
3958         # Create a collected ophandle, then collect it after 23 hours
3959         # and 59 seconds to make sure that it is still there.
3960         d = _make_collected_ophandle(133)
3961         d.addCallback(lambda ign:
3962             self.clock.advance((24*60*60) - 1))
3963         d.addCallback(lambda ign:
3964             self.GET("/operations/133?t=status&output=JSON"))
3965         def _check1(res):
3966             data = simplejson.loads(res)
3967             self.failUnless("finished" in data, res)
3968         d.addCallback(_check1)
3969         # Create another uncollected ophandle, then try to collect it
3970         # after 24 hours to make sure that it is gone.
3971         d.addCallback(lambda ign:
3972             _make_collected_ophandle(134))
3973         d.addCallback(lambda ign:
3974             self.clock.advance(24*60*60))
3975         d.addCallback(lambda ign:
3976             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3977                                  404, "404 Not Found",
3978                                  "unknown/expired handle '134'",
3979                                  self.GET,
3980                                  "/operations/134?t=status&output=JSON"))
3981         return d
3982
3983     def test_incident(self):
3984         d = self.POST("/report_incident", details="eek")
3985         def _done(res):
3986             self.failUnless("Thank you for your report!" in res, res)
3987         d.addCallback(_done)
3988         return d
3989
3990     def test_static(self):
3991         webdir = os.path.join(self.staticdir, "subdir")
3992         fileutil.make_dirs(webdir)
3993         f = open(os.path.join(webdir, "hello.txt"), "wb")
3994         f.write("hello")
3995         f.close()
3996
3997         d = self.GET("/static/subdir/hello.txt")
3998         def _check(res):
3999             self.failUnlessReallyEqual(res, "hello")
4000         d.addCallback(_check)
4001         return d
4002
4003
4004 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4005     def test_load_file(self):
4006         # This will raise an exception unless a well-formed XML file is found under that name.
4007         common.getxmlfile('directory.xhtml').load()
4008
4009     def test_parse_replace_arg(self):
4010         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4011         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4012         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4013                                    "only-files")
4014         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4015                         common.parse_replace_arg, "only_fles")
4016
4017     def test_abbreviate_time(self):
4018         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4019         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4020         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4021         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4022         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4023         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4024
4025     def test_compute_rate(self):
4026         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4027         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4028         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4029         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4030         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4031         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4032         self.shouldFail(AssertionError, "test_compute_rate", "",
4033                         common.compute_rate, -100, 10)
4034         self.shouldFail(AssertionError, "test_compute_rate", "",
4035                         common.compute_rate, 100, -10)
4036
4037         # Sanity check
4038         rate = common.compute_rate(10*1000*1000, 1)
4039         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4040
4041     def test_abbreviate_rate(self):
4042         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4043         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4044         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4045         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4046
4047     def test_abbreviate_size(self):
4048         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4049         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4050         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4051         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4052         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4053
4054     def test_plural(self):
4055         def convert(s):
4056             return "%d second%s" % (s, status.plural(s))
4057         self.failUnlessReallyEqual(convert(0), "0 seconds")
4058         self.failUnlessReallyEqual(convert(1), "1 second")
4059         self.failUnlessReallyEqual(convert(2), "2 seconds")
4060         def convert2(s):
4061             return "has share%s: %s" % (status.plural(s), ",".join(s))
4062         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4063         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4064         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4065
4066
4067 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4068
4069     def CHECK(self, ign, which, args, clientnum=0):
4070         fileurl = self.fileurls[which]
4071         url = fileurl + "?" + args
4072         return self.GET(url, method="POST", clientnum=clientnum)
4073
4074     def test_filecheck(self):
4075         self.basedir = "web/Grid/filecheck"
4076         self.set_up_grid()
4077         c0 = self.g.clients[0]
4078         self.uris = {}
4079         DATA = "data" * 100
4080         d = c0.upload(upload.Data(DATA, convergence=""))
4081         def _stash_uri(ur, which):
4082             self.uris[which] = ur.uri
4083         d.addCallback(_stash_uri, "good")
4084         d.addCallback(lambda ign:
4085                       c0.upload(upload.Data(DATA+"1", convergence="")))
4086         d.addCallback(_stash_uri, "sick")
4087         d.addCallback(lambda ign:
4088                       c0.upload(upload.Data(DATA+"2", convergence="")))
4089         d.addCallback(_stash_uri, "dead")
4090         def _stash_mutable_uri(n, which):
4091             self.uris[which] = n.get_uri()
4092             assert isinstance(self.uris[which], str)
4093         d.addCallback(lambda ign:
4094             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4095         d.addCallback(_stash_mutable_uri, "corrupt")
4096         d.addCallback(lambda ign:
4097                       c0.upload(upload.Data("literal", convergence="")))
4098         d.addCallback(_stash_uri, "small")
4099         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4100         d.addCallback(_stash_mutable_uri, "smalldir")
4101
4102         def _compute_fileurls(ignored):
4103             self.fileurls = {}
4104             for which in self.uris:
4105                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4106         d.addCallback(_compute_fileurls)
4107
4108         def _clobber_shares(ignored):
4109             good_shares = self.find_uri_shares(self.uris["good"])
4110             self.failUnlessReallyEqual(len(good_shares), 10)
4111             sick_shares = self.find_uri_shares(self.uris["sick"])
4112             os.unlink(sick_shares[0][2])
4113             dead_shares = self.find_uri_shares(self.uris["dead"])
4114             for i in range(1, 10):
4115                 os.unlink(dead_shares[i][2])
4116             c_shares = self.find_uri_shares(self.uris["corrupt"])
4117             cso = CorruptShareOptions()
4118             cso.stdout = StringIO()
4119             cso.parseOptions([c_shares[0][2]])
4120             corrupt_share(cso)
4121         d.addCallback(_clobber_shares)
4122
4123         d.addCallback(self.CHECK, "good", "t=check")
4124         def _got_html_good(res):
4125             self.failUnless("Healthy" in res, res)
4126             self.failIf("Not Healthy" in res, res)
4127         d.addCallback(_got_html_good)
4128         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4129         def _got_html_good_return_to(res):
4130             self.failUnless("Healthy" in res, res)
4131             self.failIf("Not Healthy" in res, res)
4132             self.failUnless('<a href="somewhere">Return to file'
4133                             in res, res)
4134         d.addCallback(_got_html_good_return_to)
4135         d.addCallback(self.CHECK, "good", "t=check&output=json")
4136         def _got_json_good(res):
4137             r = simplejson.loads(res)
4138             self.failUnlessEqual(r["summary"], "Healthy")
4139             self.failUnless(r["results"]["healthy"])
4140             self.failIf(r["results"]["needs-rebalancing"])
4141             self.failUnless(r["results"]["recoverable"])
4142         d.addCallback(_got_json_good)
4143
4144         d.addCallback(self.CHECK, "small", "t=check")
4145         def _got_html_small(res):
4146             self.failUnless("Literal files are always healthy" in res, res)
4147             self.failIf("Not Healthy" in res, res)
4148         d.addCallback(_got_html_small)
4149         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4150         def _got_html_small_return_to(res):
4151             self.failUnless("Literal files are always healthy" in res, res)
4152             self.failIf("Not Healthy" in res, res)
4153             self.failUnless('<a href="somewhere">Return to file'
4154                             in res, res)
4155         d.addCallback(_got_html_small_return_to)
4156         d.addCallback(self.CHECK, "small", "t=check&output=json")
4157         def _got_json_small(res):
4158             r = simplejson.loads(res)
4159             self.failUnlessEqual(r["storage-index"], "")
4160             self.failUnless(r["results"]["healthy"])
4161         d.addCallback(_got_json_small)
4162
4163         d.addCallback(self.CHECK, "smalldir", "t=check")
4164         def _got_html_smalldir(res):
4165             self.failUnless("Literal files are always healthy" in res, res)
4166             self.failIf("Not Healthy" in res, res)
4167         d.addCallback(_got_html_smalldir)
4168         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4169         def _got_json_smalldir(res):
4170             r = simplejson.loads(res)
4171             self.failUnlessEqual(r["storage-index"], "")
4172             self.failUnless(r["results"]["healthy"])
4173         d.addCallback(_got_json_smalldir)
4174
4175         d.addCallback(self.CHECK, "sick", "t=check")
4176         def _got_html_sick(res):
4177             self.failUnless("Not Healthy" in res, res)
4178         d.addCallback(_got_html_sick)
4179         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4180         def _got_json_sick(res):
4181             r = simplejson.loads(res)
4182             self.failUnlessEqual(r["summary"],
4183                                  "Not Healthy: 9 shares (enc 3-of-10)")
4184             self.failIf(r["results"]["healthy"])
4185             self.failIf(r["results"]["needs-rebalancing"])
4186             self.failUnless(r["results"]["recoverable"])
4187         d.addCallback(_got_json_sick)
4188
4189         d.addCallback(self.CHECK, "dead", "t=check")
4190         def _got_html_dead(res):
4191             self.failUnless("Not Healthy" in res, res)
4192         d.addCallback(_got_html_dead)
4193         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4194         def _got_json_dead(res):
4195             r = simplejson.loads(res)
4196             self.failUnlessEqual(r["summary"],
4197                                  "Not Healthy: 1 shares (enc 3-of-10)")
4198             self.failIf(r["results"]["healthy"])
4199             self.failIf(r["results"]["needs-rebalancing"])
4200             self.failIf(r["results"]["recoverable"])
4201         d.addCallback(_got_json_dead)
4202
4203         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4204         def _got_html_corrupt(res):
4205             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4206         d.addCallback(_got_html_corrupt)
4207         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4208         def _got_json_corrupt(res):
4209             r = simplejson.loads(res)
4210             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4211                             r["summary"])
4212             self.failIf(r["results"]["healthy"])
4213             self.failUnless(r["results"]["recoverable"])
4214             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4215             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4216         d.addCallback(_got_json_corrupt)
4217
4218         d.addErrback(self.explain_web_error)
4219         return d
4220
4221     def test_repair_html(self):
4222         self.basedir = "web/Grid/repair_html"
4223         self.set_up_grid()
4224         c0 = self.g.clients[0]
4225         self.uris = {}
4226         DATA = "data" * 100
4227         d = c0.upload(upload.Data(DATA, convergence=""))
4228         def _stash_uri(ur, which):
4229             self.uris[which] = ur.uri
4230         d.addCallback(_stash_uri, "good")
4231         d.addCallback(lambda ign:
4232                       c0.upload(upload.Data(DATA+"1", convergence="")))
4233         d.addCallback(_stash_uri, "sick")
4234         d.addCallback(lambda ign:
4235                       c0.upload(upload.Data(DATA+"2", convergence="")))
4236         d.addCallback(_stash_uri, "dead")
4237         def _stash_mutable_uri(n, which):
4238             self.uris[which] = n.get_uri()
4239             assert isinstance(self.uris[which], str)
4240         d.addCallback(lambda ign:
4241             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4242         d.addCallback(_stash_mutable_uri, "corrupt")
4243
4244         def _compute_fileurls(ignored):
4245             self.fileurls = {}
4246             for which in self.uris:
4247                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4248         d.addCallback(_compute_fileurls)
4249
4250         def _clobber_shares(ignored):
4251             good_shares = self.find_uri_shares(self.uris["good"])
4252             self.failUnlessReallyEqual(len(good_shares), 10)
4253             sick_shares = self.find_uri_shares(self.uris["sick"])
4254             os.unlink(sick_shares[0][2])
4255             dead_shares = self.find_uri_shares(self.uris["dead"])
4256             for i in range(1, 10):
4257                 os.unlink(dead_shares[i][2])
4258             c_shares = self.find_uri_shares(self.uris["corrupt"])
4259             cso = CorruptShareOptions()
4260             cso.stdout = StringIO()
4261             cso.parseOptions([c_shares[0][2]])
4262             corrupt_share(cso)
4263         d.addCallback(_clobber_shares)
4264
4265         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4266         def _got_html_good(res):
4267             self.failUnless("Healthy" in res, res)
4268             self.failIf("Not Healthy" in res, res)
4269             self.failUnless("No repair necessary" in res, res)
4270         d.addCallback(_got_html_good)
4271
4272         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4273         def _got_html_sick(res):
4274             self.failUnless("Healthy : healthy" in res, res)
4275             self.failIf("Not Healthy" in res, res)
4276             self.failUnless("Repair successful" in res, res)
4277         d.addCallback(_got_html_sick)
4278
4279         # repair of a dead file will fail, of course, but it isn't yet
4280         # clear how this should be reported. Right now it shows up as
4281         # a "410 Gone".
4282         #
4283         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4284         #def _got_html_dead(res):
4285         #    print res
4286         #    self.failUnless("Healthy : healthy" in res, res)
4287         #    self.failIf("Not Healthy" in res, res)
4288         #    self.failUnless("No repair necessary" in res, res)
4289         #d.addCallback(_got_html_dead)
4290
4291         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4292         def _got_html_corrupt(res):
4293             self.failUnless("Healthy : Healthy" in res, res)
4294             self.failIf("Not Healthy" in res, res)
4295             self.failUnless("Repair successful" in res, res)
4296         d.addCallback(_got_html_corrupt)
4297
4298         d.addErrback(self.explain_web_error)
4299         return d
4300
4301     def test_repair_json(self):
4302         self.basedir = "web/Grid/repair_json"
4303         self.set_up_grid()
4304         c0 = self.g.clients[0]
4305         self.uris = {}
4306         DATA = "data" * 100
4307         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4308         def _stash_uri(ur, which):
4309             self.uris[which] = ur.uri
4310         d.addCallback(_stash_uri, "sick")
4311
4312         def _compute_fileurls(ignored):
4313             self.fileurls = {}
4314             for which in self.uris:
4315                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4316         d.addCallback(_compute_fileurls)
4317
4318         def _clobber_shares(ignored):
4319             sick_shares = self.find_uri_shares(self.uris["sick"])
4320             os.unlink(sick_shares[0][2])
4321         d.addCallback(_clobber_shares)
4322
4323         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4324         def _got_json_sick(res):
4325             r = simplejson.loads(res)
4326             self.failUnlessReallyEqual(r["repair-attempted"], True)
4327             self.failUnlessReallyEqual(r["repair-successful"], True)
4328             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4329                                  "Not Healthy: 9 shares (enc 3-of-10)")
4330             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4331             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4332             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4333         d.addCallback(_got_json_sick)
4334
4335         d.addErrback(self.explain_web_error)
4336         return d
4337
4338     def test_unknown(self, immutable=False):
4339         self.basedir = "web/Grid/unknown"
4340         if immutable:
4341             self.basedir = "web/Grid/unknown-immutable"
4342
4343         self.set_up_grid()
4344         c0 = self.g.clients[0]
4345         self.uris = {}
4346         self.fileurls = {}
4347
4348         # the future cap format may contain slashes, which must be tolerated
4349         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4350                                                            safe="")
4351
4352         if immutable:
4353             name = u"future-imm"
4354             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4355             d = c0.create_immutable_dirnode({name: (future_node, {})})
4356         else:
4357             name = u"future"
4358             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4359             d = c0.create_dirnode()
4360
4361         def _stash_root_and_create_file(n):
4362             self.rootnode = n
4363             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4364             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4365             if not immutable:
4366                 return self.rootnode.set_node(name, future_node)
4367         d.addCallback(_stash_root_and_create_file)
4368
4369         # make sure directory listing tolerates unknown nodes
4370         d.addCallback(lambda ign: self.GET(self.rooturl))
4371         def _check_directory_html(res, expected_type_suffix):
4372             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4373                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4374                                  re.DOTALL)
4375             self.failUnless(re.search(pattern, res), res)
4376             # find the More Info link for name, should be relative
4377             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4378             info_url = mo.group(1)
4379             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4380         if immutable:
4381             d.addCallback(_check_directory_html, "-IMM")
4382         else:
4383             d.addCallback(_check_directory_html, "")
4384
4385         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4386         def _check_directory_json(res, expect_rw_uri):
4387             data = simplejson.loads(res)
4388             self.failUnlessEqual(data[0], "dirnode")
4389             f = data[1]["children"][name]
4390             self.failUnlessEqual(f[0], "unknown")
4391             if expect_rw_uri:
4392                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4393             else:
4394                 self.failIfIn("rw_uri", f[1])
4395             if immutable:
4396                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4397             else:
4398                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4399             self.failUnless("metadata" in f[1])
4400         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4401
4402         def _check_info(res, expect_rw_uri, expect_ro_uri):
4403             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4404             if expect_rw_uri:
4405                 self.failUnlessIn(unknown_rwcap, res)
4406             if expect_ro_uri:
4407                 if immutable:
4408                     self.failUnlessIn(unknown_immcap, res)
4409                 else:
4410                     self.failUnlessIn(unknown_rocap, res)
4411             else:
4412                 self.failIfIn(unknown_rocap, res)
4413             self.failIfIn("Raw data as", res)
4414             self.failIfIn("Directory writecap", res)
4415             self.failIfIn("Checker Operations", res)
4416             self.failIfIn("Mutable File Operations", res)
4417             self.failIfIn("Directory Operations", res)
4418
4419         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4420         # why they fail. Possibly related to ticket #922.
4421
4422         d.addCallback(lambda ign: self.GET(expected_info_url))
4423         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4424         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4425         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4426
4427         def _check_json(res, expect_rw_uri):
4428             data = simplejson.loads(res)
4429             self.failUnlessEqual(data[0], "unknown")
4430             if expect_rw_uri:
4431                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4432             else:
4433                 self.failIfIn("rw_uri", data[1])
4434
4435             if immutable:
4436                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4437                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4438             elif expect_rw_uri:
4439                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4440                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4441             else:
4442                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4443                 self.failIf("mutable" in data[1], data[1])
4444
4445             # TODO: check metadata contents
4446             self.failUnless("metadata" in data[1])
4447
4448         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4449         d.addCallback(_check_json, expect_rw_uri=not immutable)
4450
4451         # and make sure that a read-only version of the directory can be
4452         # rendered too. This version will not have unknown_rwcap, whether
4453         # or not future_node was immutable.
4454         d.addCallback(lambda ign: self.GET(self.rourl))
4455         if immutable:
4456             d.addCallback(_check_directory_html, "-IMM")
4457         else:
4458             d.addCallback(_check_directory_html, "-RO")
4459
4460         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4461         d.addCallback(_check_directory_json, expect_rw_uri=False)
4462
4463         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4464         d.addCallback(_check_json, expect_rw_uri=False)
4465
4466         # TODO: check that getting t=info from the Info link in the ro directory
4467         # works, and does not include the writecap URI.
4468         return d
4469
4470     def test_immutable_unknown(self):
4471         return self.test_unknown(immutable=True)
4472
4473     def test_mutant_dirnodes_are_omitted(self):
4474         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4475
4476         self.set_up_grid()
4477         c = self.g.clients[0]
4478         nm = c.nodemaker
4479         self.uris = {}
4480         self.fileurls = {}
4481
4482         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4483         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4484         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4485
4486         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4487         # test the dirnode and web layers separately.
4488
4489         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4490         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4491         # When the directory is read, the mutants should be silently disposed of, leaving
4492         # their lonely sibling.
4493         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4494         # because immutable directories don't have a writecap and therefore that field
4495         # isn't (and can't be) decrypted.
4496         # TODO: The field still exists in the netstring. Technically we should check what
4497         # happens if something is put there (_unpack_contents should raise ValueError),
4498         # but that can wait.
4499
4500         lonely_child = nm.create_from_cap(lonely_uri)
4501         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4502         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4503
4504         def _by_hook_or_by_crook():
4505             return True
4506         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4507             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4508
4509         mutant_write_in_ro_child.get_write_uri    = lambda: None
4510         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4511
4512         kids = {u"lonely":      (lonely_child, {}),
4513                 u"ro":          (mutant_ro_child, {}),
4514                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4515                 }
4516         d = c.create_immutable_dirnode(kids)
4517
4518         def _created(dn):
4519             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4520             self.failIf(dn.is_mutable())
4521             self.failUnless(dn.is_readonly())
4522             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4523             self.failIf(hasattr(dn._node, 'get_writekey'))
4524             rep = str(dn)
4525             self.failUnless("RO-IMM" in rep)
4526             cap = dn.get_cap()
4527             self.failUnlessIn("CHK", cap.to_string())
4528             self.cap = cap
4529             self.rootnode = dn
4530             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4531             return download_to_data(dn._node)
4532         d.addCallback(_created)
4533
4534         def _check_data(data):
4535             # Decode the netstring representation of the directory to check that all children
4536             # are present. This is a bit of an abstraction violation, but there's not really
4537             # any other way to do it given that the real DirectoryNode._unpack_contents would
4538             # strip the mutant children out (which is what we're trying to test, later).
4539             position = 0
4540             numkids = 0
4541             while position < len(data):
4542                 entries, position = split_netstring(data, 1, position)
4543                 entry = entries[0]
4544                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4545                 name = name_utf8.decode("utf-8")
4546                 self.failUnless(rwcapdata == "")
4547                 self.failUnless(name in kids)
4548                 (expected_child, ign) = kids[name]
4549                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4550                 numkids += 1
4551
4552             self.failUnlessReallyEqual(numkids, 3)
4553             return self.rootnode.list()
4554         d.addCallback(_check_data)
4555
4556         # Now when we use the real directory listing code, the mutants should be absent.
4557         def _check_kids(children):
4558             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4559             lonely_node, lonely_metadata = children[u"lonely"]
4560
4561             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4562             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4563         d.addCallback(_check_kids)
4564
4565         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4566         d.addCallback(lambda n: n.list())
4567         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4568
4569         # Make sure the lonely child can be listed in HTML...
4570         d.addCallback(lambda ign: self.GET(self.rooturl))
4571         def _check_html(res):
4572             self.failIfIn("URI:SSK", res)
4573             get_lonely = "".join([r'<td>FILE</td>',
4574                                   r'\s+<td>',
4575                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4576                                   r'</td>',
4577                                   r'\s+<td align="right">%d</td>' % len("one"),
4578                                   ])
4579             self.failUnless(re.search(get_lonely, res), res)
4580
4581             # find the More Info link for name, should be relative
4582             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4583             info_url = mo.group(1)
4584             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4585         d.addCallback(_check_html)
4586
4587         # ... and in JSON.
4588         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4589         def _check_json(res):
4590             data = simplejson.loads(res)
4591             self.failUnlessEqual(data[0], "dirnode")
4592             listed_children = data[1]["children"]
4593             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4594             ll_type, ll_data = listed_children[u"lonely"]
4595             self.failUnlessEqual(ll_type, "filenode")
4596             self.failIf("rw_uri" in ll_data)
4597             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4598         d.addCallback(_check_json)
4599         return d
4600
4601     def test_deep_check(self):
4602         self.basedir = "web/Grid/deep_check"
4603         self.set_up_grid()
4604         c0 = self.g.clients[0]
4605         self.uris = {}
4606         self.fileurls = {}
4607         DATA = "data" * 100
4608         d = c0.create_dirnode()
4609         def _stash_root_and_create_file(n):
4610             self.rootnode = n
4611             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4612             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4613         d.addCallback(_stash_root_and_create_file)
4614         def _stash_uri(fn, which):
4615             self.uris[which] = fn.get_uri()
4616             return fn
4617         d.addCallback(_stash_uri, "good")
4618         d.addCallback(lambda ign:
4619                       self.rootnode.add_file(u"small",
4620                                              upload.Data("literal",
4621                                                         convergence="")))
4622         d.addCallback(_stash_uri, "small")
4623         d.addCallback(lambda ign:
4624                       self.rootnode.add_file(u"sick",
4625                                              upload.Data(DATA+"1",
4626                                                         convergence="")))
4627         d.addCallback(_stash_uri, "sick")
4628
4629         # this tests that deep-check and stream-manifest will ignore
4630         # UnknownNode instances. Hopefully this will also cover deep-stats.
4631         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4632         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4633
4634         def _clobber_shares(ignored):
4635             self.delete_shares_numbered(self.uris["sick"], [0,1])
4636         d.addCallback(_clobber_shares)
4637
4638         # root
4639         # root/good
4640         # root/small
4641         # root/sick
4642         # root/future
4643
4644         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4645         def _done(res):
4646             try:
4647                 units = [simplejson.loads(line)
4648                          for line in res.splitlines()
4649                          if line]
4650             except ValueError:
4651                 print "response is:", res
4652                 print "undecodeable line was '%s'" % line
4653                 raise
4654             self.failUnlessReallyEqual(len(units), 5+1)
4655             # should be parent-first
4656             u0 = units[0]
4657             self.failUnlessEqual(u0["path"], [])
4658             self.failUnlessEqual(u0["type"], "directory")
4659             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4660             u0cr = u0["check-results"]
4661             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4662
4663             ugood = [u for u in units
4664                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4665             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4666             ugoodcr = ugood["check-results"]
4667             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4668
4669             stats = units[-1]
4670             self.failUnlessEqual(stats["type"], "stats")
4671             s = stats["stats"]
4672             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4673             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4674             self.failUnlessReallyEqual(s["count-directories"], 1)
4675             self.failUnlessReallyEqual(s["count-unknown"], 1)
4676         d.addCallback(_done)
4677
4678         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4679         def _check_manifest(res):
4680             self.failUnless(res.endswith("\n"))
4681             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4682             self.failUnlessReallyEqual(len(units), 5+1)
4683             self.failUnlessEqual(units[-1]["type"], "stats")
4684             first = units[0]
4685             self.failUnlessEqual(first["path"], [])
4686             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4687             self.failUnlessEqual(first["type"], "directory")
4688             stats = units[-1]["stats"]
4689             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4690             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4691             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4692             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4693             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4694         d.addCallback(_check_manifest)
4695
4696         # now add root/subdir and root/subdir/grandchild, then make subdir
4697         # unrecoverable, then see what happens
4698
4699         d.addCallback(lambda ign:
4700                       self.rootnode.create_subdirectory(u"subdir"))
4701         d.addCallback(_stash_uri, "subdir")
4702         d.addCallback(lambda subdir_node:
4703                       subdir_node.add_file(u"grandchild",
4704                                            upload.Data(DATA+"2",
4705                                                        convergence="")))
4706         d.addCallback(_stash_uri, "grandchild")
4707
4708         d.addCallback(lambda ign:
4709                       self.delete_shares_numbered(self.uris["subdir"],
4710                                                   range(1, 10)))
4711
4712         # root
4713         # root/good
4714         # root/small
4715         # root/sick
4716         # root/future
4717         # root/subdir [unrecoverable]
4718         # root/subdir/grandchild
4719
4720         # how should a streaming-JSON API indicate fatal error?
4721         # answer: emit ERROR: instead of a JSON string
4722
4723         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4724         def _check_broken_manifest(res):
4725             lines = res.splitlines()
4726             error_lines = [i
4727                            for (i,line) in enumerate(lines)
4728                            if line.startswith("ERROR:")]
4729             if not error_lines:
4730                 self.fail("no ERROR: in output: %s" % (res,))
4731             first_error = error_lines[0]
4732             error_line = lines[first_error]
4733             error_msg = lines[first_error+1:]
4734             error_msg_s = "\n".join(error_msg) + "\n"
4735             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4736                               error_line)
4737             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4738             units = [simplejson.loads(line) for line in lines[:first_error]]
4739             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4740             last_unit = units[-1]
4741             self.failUnlessEqual(last_unit["path"], ["subdir"])
4742         d.addCallback(_check_broken_manifest)
4743
4744         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4745         def _check_broken_deepcheck(res):
4746             lines = res.splitlines()
4747             error_lines = [i
4748                            for (i,line) in enumerate(lines)
4749                            if line.startswith("ERROR:")]
4750             if not error_lines:
4751                 self.fail("no ERROR: in output: %s" % (res,))
4752             first_error = error_lines[0]
4753             error_line = lines[first_error]
4754             error_msg = lines[first_error+1:]
4755             error_msg_s = "\n".join(error_msg) + "\n"
4756             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4757                               error_line)
4758             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4759             units = [simplejson.loads(line) for line in lines[:first_error]]
4760             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4761             last_unit = units[-1]
4762             self.failUnlessEqual(last_unit["path"], ["subdir"])
4763             r = last_unit["check-results"]["results"]
4764             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4765             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4766             self.failUnlessReallyEqual(r["recoverable"], False)
4767         d.addCallback(_check_broken_deepcheck)
4768
4769         d.addErrback(self.explain_web_error)
4770         return d
4771
4772     def test_deep_check_and_repair(self):
4773         self.basedir = "web/Grid/deep_check_and_repair"
4774         self.set_up_grid()
4775         c0 = self.g.clients[0]
4776         self.uris = {}
4777         self.fileurls = {}
4778         DATA = "data" * 100
4779         d = c0.create_dirnode()
4780         def _stash_root_and_create_file(n):
4781             self.rootnode = n
4782             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4783             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4784         d.addCallback(_stash_root_and_create_file)
4785         def _stash_uri(fn, which):
4786             self.uris[which] = fn.get_uri()
4787         d.addCallback(_stash_uri, "good")
4788         d.addCallback(lambda ign:
4789                       self.rootnode.add_file(u"small",
4790                                              upload.Data("literal",
4791                                                         convergence="")))
4792         d.addCallback(_stash_uri, "small")
4793         d.addCallback(lambda ign:
4794                       self.rootnode.add_file(u"sick",
4795                                              upload.Data(DATA+"1",
4796                                                         convergence="")))
4797         d.addCallback(_stash_uri, "sick")
4798         #d.addCallback(lambda ign:
4799         #              self.rootnode.add_file(u"dead",
4800         #                                     upload.Data(DATA+"2",
4801         #                                                convergence="")))
4802         #d.addCallback(_stash_uri, "dead")
4803
4804         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4805         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4806         #d.addCallback(_stash_uri, "corrupt")
4807
4808         def _clobber_shares(ignored):
4809             good_shares = self.find_uri_shares(self.uris["good"])
4810             self.failUnlessReallyEqual(len(good_shares), 10)
4811             sick_shares = self.find_uri_shares(self.uris["sick"])
4812             os.unlink(sick_shares[0][2])
4813             #dead_shares = self.find_uri_shares(self.uris["dead"])
4814             #for i in range(1, 10):
4815             #    os.unlink(dead_shares[i][2])
4816
4817             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4818             #cso = CorruptShareOptions()
4819             #cso.stdout = StringIO()
4820             #cso.parseOptions([c_shares[0][2]])
4821             #corrupt_share(cso)
4822         d.addCallback(_clobber_shares)
4823
4824         # root
4825         # root/good   CHK, 10 shares
4826         # root/small  LIT
4827         # root/sick   CHK, 9 shares
4828
4829         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4830         def _done(res):
4831             units = [simplejson.loads(line)
4832                      for line in res.splitlines()
4833                      if line]
4834             self.failUnlessReallyEqual(len(units), 4+1)
4835             # should be parent-first
4836             u0 = units[0]
4837             self.failUnlessEqual(u0["path"], [])
4838             self.failUnlessEqual(u0["type"], "directory")
4839             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4840             u0crr = u0["check-and-repair-results"]
4841             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4842             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4843
4844             ugood = [u for u in units
4845                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4846             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4847             ugoodcrr = ugood["check-and-repair-results"]
4848             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4849             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4850
4851             usick = [u for u in units
4852                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4853             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4854             usickcrr = usick["check-and-repair-results"]
4855             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4856             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4857             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4858             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4859
4860             stats = units[-1]
4861             self.failUnlessEqual(stats["type"], "stats")
4862             s = stats["stats"]
4863             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4864             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4865             self.failUnlessReallyEqual(s["count-directories"], 1)
4866         d.addCallback(_done)
4867
4868         d.addErrback(self.explain_web_error)
4869         return d
4870
4871     def _count_leases(self, ignored, which):
4872         u = self.uris[which]
4873         shares = self.find_uri_shares(u)
4874         lease_counts = []
4875         for shnum, serverid, fn in shares:
4876             sf = get_share_file(fn)
4877             num_leases = len(list(sf.get_leases()))
4878             lease_counts.append( (fn, num_leases) )
4879         return lease_counts
4880
4881     def _assert_leasecount(self, lease_counts, expected):
4882         for (fn, num_leases) in lease_counts:
4883             if num_leases != expected:
4884                 self.fail("expected %d leases, have %d, on %s" %
4885                           (expected, num_leases, fn))
4886
4887     def test_add_lease(self):
4888         self.basedir = "web/Grid/add_lease"
4889         self.set_up_grid(num_clients=2)
4890         c0 = self.g.clients[0]
4891         self.uris = {}
4892         DATA = "data" * 100
4893         d = c0.upload(upload.Data(DATA, convergence=""))
4894         def _stash_uri(ur, which):
4895             self.uris[which] = ur.uri
4896         d.addCallback(_stash_uri, "one")
4897         d.addCallback(lambda ign:
4898                       c0.upload(upload.Data(DATA+"1", convergence="")))
4899         d.addCallback(_stash_uri, "two")
4900         def _stash_mutable_uri(n, which):
4901             self.uris[which] = n.get_uri()
4902             assert isinstance(self.uris[which], str)
4903         d.addCallback(lambda ign:
4904             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4905         d.addCallback(_stash_mutable_uri, "mutable")
4906
4907         def _compute_fileurls(ignored):
4908             self.fileurls = {}
4909             for which in self.uris:
4910                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4911         d.addCallback(_compute_fileurls)
4912
4913         d.addCallback(self._count_leases, "one")
4914         d.addCallback(self._assert_leasecount, 1)
4915         d.addCallback(self._count_leases, "two")
4916         d.addCallback(self._assert_leasecount, 1)
4917         d.addCallback(self._count_leases, "mutable")
4918         d.addCallback(self._assert_leasecount, 1)
4919
4920         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4921         def _got_html_good(res):
4922             self.failUnless("Healthy" in res, res)
4923             self.failIf("Not Healthy" in res, res)
4924         d.addCallback(_got_html_good)
4925
4926         d.addCallback(self._count_leases, "one")
4927         d.addCallback(self._assert_leasecount, 1)
4928         d.addCallback(self._count_leases, "two")
4929         d.addCallback(self._assert_leasecount, 1)
4930         d.addCallback(self._count_leases, "mutable")
4931         d.addCallback(self._assert_leasecount, 1)
4932
4933         # this CHECK uses the original client, which uses the same
4934         # lease-secrets, so it will just renew the original lease
4935         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4936         d.addCallback(_got_html_good)
4937
4938         d.addCallback(self._count_leases, "one")
4939         d.addCallback(self._assert_leasecount, 1)
4940         d.addCallback(self._count_leases, "two")
4941         d.addCallback(self._assert_leasecount, 1)
4942         d.addCallback(self._count_leases, "mutable")
4943         d.addCallback(self._assert_leasecount, 1)
4944
4945         # this CHECK uses an alternate client, which adds a second lease
4946         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4947         d.addCallback(_got_html_good)
4948
4949         d.addCallback(self._count_leases, "one")
4950         d.addCallback(self._assert_leasecount, 2)
4951         d.addCallback(self._count_leases, "two")
4952         d.addCallback(self._assert_leasecount, 1)
4953         d.addCallback(self._count_leases, "mutable")
4954         d.addCallback(self._assert_leasecount, 1)
4955
4956         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4957         d.addCallback(_got_html_good)
4958
4959         d.addCallback(self._count_leases, "one")
4960         d.addCallback(self._assert_leasecount, 2)
4961         d.addCallback(self._count_leases, "two")
4962         d.addCallback(self._assert_leasecount, 1)
4963         d.addCallback(self._count_leases, "mutable")
4964         d.addCallback(self._assert_leasecount, 1)
4965
4966         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4967                       clientnum=1)
4968         d.addCallback(_got_html_good)
4969
4970         d.addCallback(self._count_leases, "one")
4971         d.addCallback(self._assert_leasecount, 2)
4972         d.addCallback(self._count_leases, "two")
4973         d.addCallback(self._assert_leasecount, 1)
4974         d.addCallback(self._count_leases, "mutable")
4975         d.addCallback(self._assert_leasecount, 2)
4976
4977         d.addErrback(self.explain_web_error)
4978         return d
4979
4980     def test_deep_add_lease(self):
4981         self.basedir = "web/Grid/deep_add_lease"
4982         self.set_up_grid(num_clients=2)
4983         c0 = self.g.clients[0]
4984         self.uris = {}
4985         self.fileurls = {}
4986         DATA = "data" * 100
4987         d = c0.create_dirnode()
4988         def _stash_root_and_create_file(n):
4989             self.rootnode = n
4990             self.uris["root"] = n.get_uri()
4991             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4992             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4993         d.addCallback(_stash_root_and_create_file)
4994         def _stash_uri(fn, which):
4995             self.uris[which] = fn.get_uri()
4996         d.addCallback(_stash_uri, "one")
4997         d.addCallback(lambda ign:
4998                       self.rootnode.add_file(u"small",
4999                                              upload.Data("literal",
5000                                                         convergence="")))
5001         d.addCallback(_stash_uri, "small")
5002
5003         d.addCallback(lambda ign:
5004             c0.create_mutable_file(publish.MutableData("mutable")))
5005         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5006         d.addCallback(_stash_uri, "mutable")
5007
5008         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5009         def _done(res):
5010             units = [simplejson.loads(line)
5011                      for line in res.splitlines()
5012                      if line]
5013             # root, one, small, mutable,   stats
5014             self.failUnlessReallyEqual(len(units), 4+1)
5015         d.addCallback(_done)
5016
5017         d.addCallback(self._count_leases, "root")
5018         d.addCallback(self._assert_leasecount, 1)
5019         d.addCallback(self._count_leases, "one")
5020         d.addCallback(self._assert_leasecount, 1)
5021         d.addCallback(self._count_leases, "mutable")
5022         d.addCallback(self._assert_leasecount, 1)
5023
5024         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5025         d.addCallback(_done)
5026
5027         d.addCallback(self._count_leases, "root")
5028         d.addCallback(self._assert_leasecount, 1)
5029         d.addCallback(self._count_leases, "one")
5030         d.addCallback(self._assert_leasecount, 1)
5031         d.addCallback(self._count_leases, "mutable")
5032         d.addCallback(self._assert_leasecount, 1)
5033
5034         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5035                       clientnum=1)
5036         d.addCallback(_done)
5037
5038         d.addCallback(self._count_leases, "root")
5039         d.addCallback(self._assert_leasecount, 2)
5040         d.addCallback(self._count_leases, "one")
5041         d.addCallback(self._assert_leasecount, 2)
5042         d.addCallback(self._count_leases, "mutable")
5043         d.addCallback(self._assert_leasecount, 2)
5044
5045         d.addErrback(self.explain_web_error)
5046         return d
5047
5048
5049     def test_exceptions(self):
5050         self.basedir = "web/Grid/exceptions"
5051         self.set_up_grid(num_clients=1, num_servers=2)
5052         c0 = self.g.clients[0]
5053         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5054         self.fileurls = {}
5055         DATA = "data" * 100
5056         d = c0.create_dirnode()
5057         def _stash_root(n):
5058             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5059             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5060             return n
5061         d.addCallback(_stash_root)
5062         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5063         def _stash_bad(ur):
5064             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5065             self.delete_shares_numbered(ur.uri, range(1,10))
5066
5067             u = uri.from_string(ur.uri)
5068             u.key = testutil.flip_bit(u.key, 0)
5069             baduri = u.to_string()
5070             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5071         d.addCallback(_stash_bad)
5072         d.addCallback(lambda ign: c0.create_dirnode())
5073         def _mangle_dirnode_1share(n):
5074             u = n.get_uri()
5075             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5076             self.fileurls["dir-1share-json"] = url + "?t=json"
5077             self.delete_shares_numbered(u, range(1,10))
5078         d.addCallback(_mangle_dirnode_1share)
5079         d.addCallback(lambda ign: c0.create_dirnode())
5080         def _mangle_dirnode_0share(n):
5081             u = n.get_uri()
5082             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5083             self.fileurls["dir-0share-json"] = url + "?t=json"
5084             self.delete_shares_numbered(u, range(0,10))
5085         d.addCallback(_mangle_dirnode_0share)
5086
5087         # NotEnoughSharesError should be reported sensibly, with a
5088         # text/plain explanation of the problem, and perhaps some
5089         # information on which shares *could* be found.
5090
5091         d.addCallback(lambda ignored:
5092                       self.shouldHTTPError("GET unrecoverable",
5093                                            410, "Gone", "NoSharesError",
5094                                            self.GET, self.fileurls["0shares"]))
5095         def _check_zero_shares(body):
5096             self.failIf("<html>" in body, body)
5097             body = " ".join(body.strip().split())
5098             exp = ("NoSharesError: no shares could be found. "
5099                    "Zero shares usually indicates a corrupt URI, or that "
5100                    "no servers were connected, but it might also indicate "
5101                    "severe corruption. You should perform a filecheck on "
5102                    "this object to learn more. The full error message is: "
5103                    "no shares (need 3). Last failure: None")
5104             self.failUnlessReallyEqual(exp, body)
5105         d.addCallback(_check_zero_shares)
5106
5107
5108         d.addCallback(lambda ignored:
5109                       self.shouldHTTPError("GET 1share",
5110                                            410, "Gone", "NotEnoughSharesError",
5111                                            self.GET, self.fileurls["1share"]))
5112         def _check_one_share(body):
5113             self.failIf("<html>" in body, body)
5114             body = " ".join(body.strip().split())
5115             msgbase = ("NotEnoughSharesError: This indicates that some "
5116                        "servers were unavailable, or that shares have been "
5117                        "lost to server departure, hard drive failure, or disk "
5118                        "corruption. You should perform a filecheck on "
5119                        "this object to learn more. The full error message is:"
5120                        )
5121             msg1 = msgbase + (" ran out of shares:"
5122                               " complete=sh0"
5123                               " pending="
5124                               " overdue= unused= need 3. Last failure: None")
5125             msg2 = msgbase + (" ran out of shares:"
5126                               " complete="
5127                               " pending=Share(sh0-on-xgru5)"
5128                               " overdue= unused= need 3. Last failure: None")
5129             self.failUnless(body == msg1 or body == msg2, body)
5130         d.addCallback(_check_one_share)
5131
5132         d.addCallback(lambda ignored:
5133                       self.shouldHTTPError("GET imaginary",
5134                                            404, "Not Found", None,
5135                                            self.GET, self.fileurls["imaginary"]))
5136         def _missing_child(body):
5137             self.failUnless("No such child: imaginary" in body, body)
5138         d.addCallback(_missing_child)
5139
5140         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5141         def _check_0shares_dir_html(body):
5142             self.failUnless("<html>" in body, body)
5143             # we should see the regular page, but without the child table or
5144             # the dirops forms
5145             body = " ".join(body.strip().split())
5146             self.failUnlessIn('href="?t=info">More info on this directory',
5147                               body)
5148             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5149                    "could not be retrieved, because there were insufficient "
5150                    "good shares. This might indicate that no servers were "
5151                    "connected, insufficient servers were connected, the URI "
5152                    "was corrupt, or that shares have been lost due to server "
5153                    "departure, hard drive failure, or disk corruption. You "
5154                    "should perform a filecheck on this object to learn more.")
5155             self.failUnlessIn(exp, body)
5156             self.failUnlessIn("No upload forms: directory is unreadable", body)
5157         d.addCallback(_check_0shares_dir_html)
5158
5159         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5160         def _check_1shares_dir_html(body):
5161             # at some point, we'll split UnrecoverableFileError into 0-shares
5162             # and some-shares like we did for immutable files (since there
5163             # are different sorts of advice to offer in each case). For now,
5164             # they present the same way.
5165             self.failUnless("<html>" in body, body)
5166             body = " ".join(body.strip().split())
5167             self.failUnlessIn('href="?t=info">More info on this directory',
5168                               body)
5169             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5170                    "could not be retrieved, because there were insufficient "
5171                    "good shares. This might indicate that no servers were "
5172                    "connected, insufficient servers were connected, the URI "
5173                    "was corrupt, or that shares have been lost due to server "
5174                    "departure, hard drive failure, or disk corruption. You "
5175                    "should perform a filecheck on this object to learn more.")
5176             self.failUnlessIn(exp, body)
5177             self.failUnlessIn("No upload forms: directory is unreadable", body)
5178         d.addCallback(_check_1shares_dir_html)
5179
5180         d.addCallback(lambda ignored:
5181                       self.shouldHTTPError("GET dir-0share-json",
5182                                            410, "Gone", "UnrecoverableFileError",
5183                                            self.GET,
5184                                            self.fileurls["dir-0share-json"]))
5185         def _check_unrecoverable_file(body):
5186             self.failIf("<html>" in body, body)
5187             body = " ".join(body.strip().split())
5188             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5189                    "could not be retrieved, because there were insufficient "
5190                    "good shares. This might indicate that no servers were "
5191                    "connected, insufficient servers were connected, the URI "
5192                    "was corrupt, or that shares have been lost due to server "
5193                    "departure, hard drive failure, or disk corruption. You "
5194                    "should perform a filecheck on this object to learn more.")
5195             self.failUnlessReallyEqual(exp, body)
5196         d.addCallback(_check_unrecoverable_file)
5197
5198         d.addCallback(lambda ignored:
5199                       self.shouldHTTPError("GET dir-1share-json",
5200                                            410, "Gone", "UnrecoverableFileError",
5201                                            self.GET,
5202                                            self.fileurls["dir-1share-json"]))
5203         d.addCallback(_check_unrecoverable_file)
5204
5205         d.addCallback(lambda ignored:
5206                       self.shouldHTTPError("GET imaginary",
5207                                            404, "Not Found", None,
5208                                            self.GET, self.fileurls["imaginary"]))
5209
5210         # attach a webapi child that throws a random error, to test how it
5211         # gets rendered.
5212         w = c0.getServiceNamed("webish")
5213         w.root.putChild("ERRORBOOM", ErrorBoom())
5214
5215         # "Accept: */*" :        should get a text/html stack trace
5216         # "Accept: text/plain" : should get a text/plain stack trace
5217         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5218         # no Accept header:      should get a text/html stack trace
5219
5220         d.addCallback(lambda ignored:
5221                       self.shouldHTTPError("GET errorboom_html",
5222                                            500, "Internal Server Error", None,
5223                                            self.GET, "ERRORBOOM",
5224                                            headers={"accept": ["*/*"]}))
5225         def _internal_error_html1(body):
5226             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5227         d.addCallback(_internal_error_html1)
5228
5229         d.addCallback(lambda ignored:
5230                       self.shouldHTTPError("GET errorboom_text",
5231                                            500, "Internal Server Error", None,
5232                                            self.GET, "ERRORBOOM",
5233                                            headers={"accept": ["text/plain"]}))
5234         def _internal_error_text2(body):
5235             self.failIf("<html>" in body, body)
5236             self.failUnless(body.startswith("Traceback "), body)
5237         d.addCallback(_internal_error_text2)
5238
5239         CLI_accepts = "text/plain, application/octet-stream"
5240         d.addCallback(lambda ignored:
5241                       self.shouldHTTPError("GET errorboom_text",
5242                                            500, "Internal Server Error", None,
5243                                            self.GET, "ERRORBOOM",
5244                                            headers={"accept": [CLI_accepts]}))
5245         def _internal_error_text3(body):
5246             self.failIf("<html>" in body, body)
5247             self.failUnless(body.startswith("Traceback "), body)
5248         d.addCallback(_internal_error_text3)
5249
5250         d.addCallback(lambda ignored:
5251                       self.shouldHTTPError("GET errorboom_text",
5252                                            500, "Internal Server Error", None,
5253                                            self.GET, "ERRORBOOM"))
5254         def _internal_error_html4(body):
5255             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5256         d.addCallback(_internal_error_html4)
5257
5258         def _flush_errors(res):
5259             # Trial: please ignore the CompletelyUnhandledError in the logs
5260             self.flushLoggedErrors(CompletelyUnhandledError)
5261             return res
5262         d.addBoth(_flush_errors)
5263
5264         return d
5265
5266     def test_blacklist(self):
5267         # download from a blacklisted URI, get an error
5268         self.basedir = "web/Grid/blacklist"
5269         self.set_up_grid()
5270         c0 = self.g.clients[0]
5271         c0_basedir = c0.basedir
5272         fn = os.path.join(c0_basedir, "access.blacklist")
5273         self.uris = {}
5274         DATA = "off-limits " * 50
5275
5276         d = c0.upload(upload.Data(DATA, convergence=""))
5277         def _stash_uri_and_create_dir(ur):
5278             self.uri = ur.uri
5279             self.url = "uri/"+self.uri
5280             u = uri.from_string_filenode(self.uri)
5281             self.si = u.get_storage_index()
5282             childnode = c0.create_node_from_uri(self.uri, None)
5283             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5284         d.addCallback(_stash_uri_and_create_dir)
5285         def _stash_dir(node):
5286             self.dir_node = node
5287             self.dir_uri = node.get_uri()
5288             self.dir_url = "uri/"+self.dir_uri
5289         d.addCallback(_stash_dir)
5290         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5291         def _check_dir_html(body):
5292             self.failUnlessIn("<html>", body)
5293             self.failUnlessIn("blacklisted.txt</a>", body)
5294         d.addCallback(_check_dir_html)
5295         d.addCallback(lambda ign: self.GET(self.url))
5296         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5297
5298         def _blacklist(ign):
5299             f = open(fn, "w")
5300             f.write(" # this is a comment\n")
5301             f.write(" \n")
5302             f.write("\n") # also exercise blank lines
5303             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5304             f.close()
5305             # clients should be checking the blacklist each time, so we don't
5306             # need to restart the client
5307         d.addCallback(_blacklist)
5308         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5309                                                        403, "Forbidden",
5310                                                        "Access Prohibited: off-limits",
5311                                                        self.GET, self.url))
5312
5313         # We should still be able to list the parent directory, in HTML...
5314         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5315         def _check_dir_html2(body):
5316             self.failUnlessIn("<html>", body)
5317             self.failUnlessIn("blacklisted.txt</strike>", body)
5318         d.addCallback(_check_dir_html2)
5319
5320         # ... and in JSON (used by CLI).
5321         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5322         def _check_dir_json(res):
5323             data = simplejson.loads(res)
5324             self.failUnless(isinstance(data, list), data)
5325             self.failUnlessEqual(data[0], "dirnode")
5326             self.failUnless(isinstance(data[1], dict), data)
5327             self.failUnlessIn("children", data[1])
5328             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5329             childdata = data[1]["children"]["blacklisted.txt"]
5330             self.failUnless(isinstance(childdata, list), data)
5331             self.failUnlessEqual(childdata[0], "filenode")
5332             self.failUnless(isinstance(childdata[1], dict), data)
5333         d.addCallback(_check_dir_json)
5334
5335         def _unblacklist(ign):
5336             open(fn, "w").close()
5337             # the Blacklist object watches mtime to tell when the file has
5338             # changed, but on windows this test will run faster than the
5339             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5340             # to force a reload.
5341             self.g.clients[0].blacklist.last_mtime -= 2.0
5342         d.addCallback(_unblacklist)
5343
5344         # now a read should work
5345         d.addCallback(lambda ign: self.GET(self.url))
5346         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5347
5348         # read again to exercise the blacklist-is-unchanged logic
5349         d.addCallback(lambda ign: self.GET(self.url))
5350         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5351
5352         # now add a blacklisted directory, and make sure files under it are
5353         # refused too
5354         def _add_dir(ign):
5355             childnode = c0.create_node_from_uri(self.uri, None)
5356             return c0.create_dirnode({u"child": (childnode,{}) })
5357         d.addCallback(_add_dir)
5358         def _get_dircap(dn):
5359             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5360             self.dir_url_base = "uri/"+dn.get_write_uri()
5361             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5362             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5363             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5364             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5365         d.addCallback(_get_dircap)
5366         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5367         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5368         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5369         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5370         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5371         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5372         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5373         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5374         d.addCallback(lambda ign: self.GET(self.child_url))
5375         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5376
5377         def _block_dir(ign):
5378             f = open(fn, "w")
5379             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5380             f.close()
5381             self.g.clients[0].blacklist.last_mtime -= 2.0
5382         d.addCallback(_block_dir)
5383         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5384                                                        403, "Forbidden",
5385                                                        "Access Prohibited: dir-off-limits",
5386                                                        self.GET, self.dir_url_base))
5387         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5388                                                        403, "Forbidden",
5389                                                        "Access Prohibited: dir-off-limits",
5390                                                        self.GET, self.dir_url_json1))
5391         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5392                                                        403, "Forbidden",
5393                                                        "Access Prohibited: dir-off-limits",
5394                                                        self.GET, self.dir_url_json2))
5395         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5396                                                        403, "Forbidden",
5397                                                        "Access Prohibited: dir-off-limits",
5398                                                        self.GET, self.dir_url_json_ro))
5399         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5400                                                        403, "Forbidden",
5401                                                        "Access Prohibited: dir-off-limits",
5402                                                        self.GET, self.child_url))
5403         return d
5404
5405
5406 class CompletelyUnhandledError(Exception):
5407     pass
5408 class ErrorBoom(rend.Page):
5409     def beforeRender(self, ctx):
5410         raise CompletelyUnhandledError("whoops")