src/allmydata/test/test_web.py

   1
   2 import os.path, re, urllib, time
   3 import simplejson
   4 from StringIO import StringIO
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11 from nevow import rend
  12 from allmydata import interfaces, uri, webish, dirnode
  13 from allmydata.storage.shares import get_share_file
  14 from allmydata.storage_client import StorageFarmBroker
  15 from allmydata.immutable import upload
  16 from allmydata.immutable.downloader.status import DownloadStatus
  17 from allmydata.dirnode import DirectoryNode
  18 from allmydata.nodemaker import NodeMaker
  19 from allmydata.unknown import UnknownNode
  20 from allmydata.web import status, common
  21 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  22 from allmydata.util import fileutil, base32, hashutil
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.util.netstring import split_netstring
  25 from allmydata.util.encodingutil import to_str
  26 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  27      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  28 from allmydata.interfaces import IMutableFileNode
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     def _create_lit(self, cap):
  52         return FakeCHKFileNode(cap)
  53     def _create_immutable(self, cap):
  54         return FakeCHKFileNode(cap)
  55     def _create_mutable(self, cap):
  56         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  57     def create_mutable_file(self, contents="", keysize=None):
  58         n = FakeMutableFileNode(None, None, None, None)
  59         return n.create(contents)
  60
  61 class FakeUploader(service.Service):
  62     name = "uploader"
  63     def upload(self, uploadable, history=None):
  64         d = uploadable.get_size()
  65         d.addCallback(lambda size: uploadable.read(size))
  66         def _got_data(datav):
  67             data = "".join(datav)
  68             n = create_chk_filenode(data)
  69             results = upload.UploadResults()
  70             results.uri = n.get_uri()
  71             return results
  72         d.addCallback(_got_data)
  73         return d
  74     def get_helper_info(self):
  75         return (None, False)
  76
  77 class FakeIServer:
  78     def __init__(self, binaryserverid):
  79         self.binaryserverid = binaryserverid
  80     def get_name(self): return "short"
  81     def get_longname(self): return "long"
  82     def get_serverid(self): return self.binaryserverid
  83
  84 def build_one_ds():
  85     ds = DownloadStatus("storage_index", 1234)
  86     now = time.time()
  87
  88     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  89     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  90     storage_index = hashutil.storage_index_hash("SI")
  91     e0 = ds.add_segment_request(0, now)
  92     e0.activate(now+0.5)
  93     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
  94     e1 = ds.add_segment_request(1, now+2)
  95     e1.error(now+3)
  96     # two outstanding requests
  97     e2 = ds.add_segment_request(2, now+4)
  98     e3 = ds.add_segment_request(3, now+5)
  99     del e2,e3 # hush pyflakes
 100
 101     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 102     e = ds.add_segment_request(4, now)
 103     e.activate(now)
 104     e.deliver(now, 0, 140, 0.5)
 105
 106     e = ds.add_dyhb_request(serverA, now)
 107     e.finished([1,2], now+1)
 108     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 109
 110     e = ds.add_read_event(0, 120, now)
 111     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 112     e.finished(now+1)
 113     e = ds.add_read_event(120, 30, now+2) # left unfinished
 114
 115     e = ds.add_block_request(serverA, 1, 100, 20, now)
 116     e.finished(20, now+1)
 117     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 118
 119     # make sure that add_read_event() can come first too
 120     ds1 = DownloadStatus(storage_index, 1234)
 121     e = ds1.add_read_event(0, 120, now)
 122     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 123     e.finished(now+1)
 124
 125     return ds
 126
 127 class FakeHistory:
 128     _all_upload_status = [upload.UploadStatus()]
 129     _all_download_status = [build_one_ds()]
 130     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 131     _all_publish_statuses = [publish.PublishStatus()]
 132     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 133
 134     def list_all_upload_statuses(self):
 135         return self._all_upload_status
 136     def list_all_download_statuses(self):
 137         return self._all_download_status
 138     def list_all_mapupdate_statuses(self):
 139         return self._all_mapupdate_statuses
 140     def list_all_publish_statuses(self):
 141         return self._all_publish_statuses
 142     def list_all_retrieve_statuses(self):
 143         return self._all_retrieve_statuses
 144     def list_all_helper_statuses(self):
 145         return []
 146
 147 class FakeClient(Client):
 148     def __init__(self):
 149         # don't upcall to Client.__init__, since we only want to initialize a
 150         # minimal subset
 151         service.MultiService.__init__(self)
 152         self.nodeid = "fake_nodeid"
 153         self.nickname = "fake_nickname"
 154         self.introducer_furl = "None"
 155         self.stats_provider = FakeStatsProvider()
 156         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 157         self.helper = None
 158         self.convergence = "some random string"
 159         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 160         self.introducer_client = None
 161         self.history = FakeHistory()
 162         self.uploader = FakeUploader()
 163         self.uploader.setServiceParent(self)
 164         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 165                                        self.uploader, None,
 166                                        None, None)
 167
 168     def startService(self):
 169         return service.MultiService.startService(self)
 170     def stopService(self):
 171         return service.MultiService.stopService(self)
 172
 173     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 174
 175 class WebMixin(object):
 176     def setUp(self):
 177         self.s = FakeClient()
 178         self.s.startService()
 179         self.staticdir = self.mktemp()
 180         self.clock = Clock()
 181         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 182                                       clock=self.clock)
 183         self.ws.setServiceParent(self.s)
 184         self.webish_port = self.ws.getPortnum()
 185         self.webish_url = self.ws.getURL()
 186         assert self.webish_url.endswith("/")
 187         self.webish_url = self.webish_url[:-1] # these tests add their own /
 188
 189         l = [ self.s.create_dirnode() for x in range(6) ]
 190         d = defer.DeferredList(l)
 191         def _then(res):
 192             self.public_root = res[0][1]
 193             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 194             self.public_url = "/uri/" + self.public_root.get_uri()
 195             self.private_root = res[1][1]
 196
 197             foo = res[2][1]
 198             self._foo_node = foo
 199             self._foo_uri = foo.get_uri()
 200             self._foo_readonly_uri = foo.get_readonly_uri()
 201             self._foo_verifycap = foo.get_verify_cap().to_string()
 202             # NOTE: we ignore the deferred on all set_uri() calls, because we
 203             # know the fake nodes do these synchronously
 204             self.public_root.set_uri(u"foo", foo.get_uri(),
 205                                      foo.get_readonly_uri())
 206
 207             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 208             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 209             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 210
 211             foo.set_uri(u"empty", res[3][1].get_uri(),
 212                         res[3][1].get_readonly_uri())
 213             sub_uri = res[4][1].get_uri()
 214             self._sub_uri = sub_uri
 215             foo.set_uri(u"sub", sub_uri, sub_uri)
 216             sub = self.s.create_node_from_uri(sub_uri)
 217
 218             _ign, n, blocking_uri = self.makefile(1)
 219             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 220
 221             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 222             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 223             # still think of it as an umlaut
 224             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 225
 226             _ign, n, baz_file = self.makefile(2)
 227             self._baz_file_uri = baz_file
 228             sub.set_uri(u"baz.txt", baz_file, baz_file)
 229
 230             _ign, n, self._bad_file_uri = self.makefile(3)
 231             # this uri should not be downloadable
 232             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 233
 234             rodir = res[5][1]
 235             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 236                                      rodir.get_readonly_uri())
 237             rodir.set_uri(u"nor", baz_file, baz_file)
 238
 239             # public/
 240             # public/foo/
 241             # public/foo/bar.txt
 242             # public/foo/blockingfile
 243             # public/foo/empty/
 244             # public/foo/sub/
 245             # public/foo/sub/baz.txt
 246             # public/reedownlee/
 247             # public/reedownlee/nor
 248             self.NEWFILE_CONTENTS = "newfile contents\n"
 249
 250             return foo.get_metadata_for(u"bar.txt")
 251         d.addCallback(_then)
 252         def _got_metadata(metadata):
 253             self._bar_txt_metadata = metadata
 254         d.addCallback(_got_metadata)
 255         return d
 256
 257     def makefile(self, number):
 258         contents = "contents of file %s\n" % number
 259         n = create_chk_filenode(contents)
 260         return contents, n, n.get_uri()
 261
 262     def tearDown(self):
 263         return self.s.stopService()
 264
 265     def failUnlessIsBarDotTxt(self, res):
 266         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 267
 268     def failUnlessIsBarJSON(self, res):
 269         data = simplejson.loads(res)
 270         self.failUnless(isinstance(data, list))
 271         self.failUnlessEqual(data[0], "filenode")
 272         self.failUnless(isinstance(data[1], dict))
 273         self.failIf(data[1]["mutable"])
 274         self.failIf("rw_uri" in data[1]) # immutable
 275         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 276         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 277         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 278
 279     def failUnlessIsFooJSON(self, res):
 280         data = simplejson.loads(res)
 281         self.failUnless(isinstance(data, list))
 282         self.failUnlessEqual(data[0], "dirnode", res)
 283         self.failUnless(isinstance(data[1], dict))
 284         self.failUnless(data[1]["mutable"])
 285         self.failUnless("rw_uri" in data[1]) # mutable
 286         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 287         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 288         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 289
 290         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 291         self.failUnlessEqual(kidnames,
 292                              [u"bar.txt", u"blockingfile", u"empty",
 293                               u"n\u00fc.txt", u"sub"])
 294         kids = dict( [(unicode(name),value)
 295                       for (name,value)
 296                       in data[1]["children"].iteritems()] )
 297         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 298         self.failUnlessIn("metadata", kids[u"sub"][1])
 299         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 300         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 301         self.failUnlessIn("linkcrtime", tahoe_md)
 302         self.failUnlessIn("linkmotime", tahoe_md)
 303         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 304         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 305         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 306         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 307                                    self._bar_txt_verifycap)
 308         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 309         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 310         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 311                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 312         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 313                                    self._bar_txt_uri)
 314
 315     def GET(self, urlpath, followRedirect=False, return_response=False,
 316             **kwargs):
 317         # if return_response=True, this fires with (data, statuscode,
 318         # respheaders) instead of just data.
 319         assert not isinstance(urlpath, unicode)
 320         url = self.webish_url + urlpath
 321         factory = HTTPClientGETFactory(url, method="GET",
 322                                        followRedirect=followRedirect, **kwargs)
 323         reactor.connectTCP("localhost", self.webish_port, factory)
 324         d = factory.deferred
 325         def _got_data(data):
 326             return (data, factory.status, factory.response_headers)
 327         if return_response:
 328             d.addCallback(_got_data)
 329         return factory.deferred
 330
 331     def HEAD(self, urlpath, return_response=False, **kwargs):
 332         # this requires some surgery, because twisted.web.client doesn't want
 333         # to give us back the response headers.
 334         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 335         reactor.connectTCP("localhost", self.webish_port, factory)
 336         d = factory.deferred
 337         def _got_data(data):
 338             return (data, factory.status, factory.response_headers)
 339         if return_response:
 340             d.addCallback(_got_data)
 341         return factory.deferred
 342
 343     def PUT(self, urlpath, data, **kwargs):
 344         url = self.webish_url + urlpath
 345         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 346
 347     def DELETE(self, urlpath):
 348         url = self.webish_url + urlpath
 349         return client.getPage(url, method="DELETE")
 350
 351     def POST(self, urlpath, followRedirect=False, **fields):
 352         sepbase = "boogabooga"
 353         sep = "--" + sepbase
 354         form = []
 355         form.append(sep)
 356         form.append('Content-Disposition: form-data; name="_charset"')
 357         form.append('')
 358         form.append('UTF-8')
 359         form.append(sep)
 360         for name, value in fields.iteritems():
 361             if isinstance(value, tuple):
 362                 filename, value = value
 363                 form.append('Content-Disposition: form-data; name="%s"; '
 364                             'filename="%s"' % (name, filename.encode("utf-8")))
 365             else:
 366                 form.append('Content-Disposition: form-data; name="%s"' % name)
 367             form.append('')
 368             if isinstance(value, unicode):
 369                 value = value.encode("utf-8")
 370             else:
 371                 value = str(value)
 372             assert isinstance(value, str)
 373             form.append(value)
 374             form.append(sep)
 375         form[-1] += "--"
 376         body = ""
 377         headers = {}
 378         if fields:
 379             body = "\r\n".join(form) + "\r\n"
 380             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 381         return self.POST2(urlpath, body, headers, followRedirect)
 382
 383     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 384         url = self.webish_url + urlpath
 385         return client.getPage(url, method="POST", postdata=body,
 386                               headers=headers, followRedirect=followRedirect)
 387
 388     def shouldFail(self, res, expected_failure, which,
 389                    substring=None, response_substring=None):
 390         if isinstance(res, failure.Failure):
 391             res.trap(expected_failure)
 392             if substring:
 393                 self.failUnless(substring in str(res),
 394                                 "substring '%s' not in '%s'"
 395                                 % (substring, str(res)))
 396             if response_substring:
 397                 self.failUnless(response_substring in res.value.response,
 398                                 "response substring '%s' not in '%s'"
 399                                 % (response_substring, res.value.response))
 400         else:
 401             self.fail("%s was supposed to raise %s, not get '%s'" %
 402                       (which, expected_failure, res))
 403
 404     def shouldFail2(self, expected_failure, which, substring,
 405                     response_substring,
 406                     callable, *args, **kwargs):
 407         assert substring is None or isinstance(substring, str)
 408         assert response_substring is None or isinstance(response_substring, str)
 409         d = defer.maybeDeferred(callable, *args, **kwargs)
 410         def done(res):
 411             if isinstance(res, failure.Failure):
 412                 res.trap(expected_failure)
 413                 if substring:
 414                     self.failUnless(substring in str(res),
 415                                     "%s: substring '%s' not in '%s'"
 416                                     % (which, substring, str(res)))
 417                 if response_substring:
 418                     self.failUnless(response_substring in res.value.response,
 419                                     "%s: response substring '%s' not in '%s'"
 420                                     % (which,
 421                                        response_substring, res.value.response))
 422             else:
 423                 self.fail("%s was supposed to raise %s, not get '%s'" %
 424                           (which, expected_failure, res))
 425         d.addBoth(done)
 426         return d
 427
 428     def should404(self, res, which):
 429         if isinstance(res, failure.Failure):
 430             res.trap(error.Error)
 431             self.failUnlessReallyEqual(res.value.status, "404")
 432         else:
 433             self.fail("%s was supposed to Error(404), not get '%s'" %
 434                       (which, res))
 435
 436     def should302(self, res, which):
 437         if isinstance(res, failure.Failure):
 438             res.trap(error.Error)
 439             self.failUnlessReallyEqual(res.value.status, "302")
 440         else:
 441             self.fail("%s was supposed to Error(302), not get '%s'" %
 442                         (which, res))
 443
 444
 445 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 446     def test_create(self):
 447         pass
 448
 449     def test_welcome(self):
 450         d = self.GET("/")
 451         def _check(res):
 452             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 453
 454             self.s.basedir = 'web/test_welcome'
 455             fileutil.make_dirs("web/test_welcome")
 456             fileutil.make_dirs("web/test_welcome/private")
 457             return self.GET("/")
 458         d.addCallback(_check)
 459         return d
 460
 461     def test_provisioning(self):
 462         d = self.GET("/provisioning/")
 463         def _check(res):
 464             self.failUnless('Provisioning Tool' in res)
 465             fields = {'filled': True,
 466                       "num_users": int(50e3),
 467                       "files_per_user": 1000,
 468                       "space_per_user": int(1e9),
 469                       "sharing_ratio": 1.0,
 470                       "encoding_parameters": "3-of-10-5",
 471                       "num_servers": 30,
 472                       "ownership_mode": "A",
 473                       "download_rate": 100,
 474                       "upload_rate": 10,
 475                       "delete_rate": 10,
 476                       "lease_timer": 7,
 477                       }
 478             return self.POST("/provisioning/", **fields)
 479
 480         d.addCallback(_check)
 481         def _check2(res):
 482             self.failUnless('Provisioning Tool' in res)
 483             self.failUnless("Share space consumed: 167.01TB" in res)
 484
 485             fields = {'filled': True,
 486                       "num_users": int(50e6),
 487                       "files_per_user": 1000,
 488                       "space_per_user": int(5e9),
 489                       "sharing_ratio": 1.0,
 490                       "encoding_parameters": "25-of-100-50",
 491                       "num_servers": 30000,
 492                       "ownership_mode": "E",
 493                       "drive_failure_model": "U",
 494                       "drive_size": 1000,
 495                       "download_rate": 1000,
 496                       "upload_rate": 100,
 497                       "delete_rate": 100,
 498                       "lease_timer": 7,
 499                       }
 500             return self.POST("/provisioning/", **fields)
 501         d.addCallback(_check2)
 502         def _check3(res):
 503             self.failUnless("Share space consumed: huge!" in res)
 504             fields = {'filled': True}
 505             return self.POST("/provisioning/", **fields)
 506         d.addCallback(_check3)
 507         def _check4(res):
 508             self.failUnless("Share space consumed:" in res)
 509         d.addCallback(_check4)
 510         return d
 511
 512     def test_reliability_tool(self):
 513         try:
 514             from allmydata import reliability
 515             _hush_pyflakes = reliability
 516             del _hush_pyflakes
 517         except:
 518             raise unittest.SkipTest("reliability tool requires NumPy")
 519
 520         d = self.GET("/reliability/")
 521         def _check(res):
 522             self.failUnless('Reliability Tool' in res)
 523             fields = {'drive_lifetime': "8Y",
 524                       "k": "3",
 525                       "R": "7",
 526                       "N": "10",
 527                       "delta": "100000",
 528                       "check_period": "1M",
 529                       "report_period": "3M",
 530                       "report_span": "5Y",
 531                       }
 532             return self.POST("/reliability/", **fields)
 533
 534         d.addCallback(_check)
 535         def _check2(res):
 536             self.failUnless('Reliability Tool' in res)
 537             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 538             self.failUnless(re.search(r, res), res)
 539         d.addCallback(_check2)
 540         return d
 541
 542     def test_status(self):
 543         h = self.s.get_history()
 544         dl_num = h.list_all_download_statuses()[0].get_counter()
 545         ul_num = h.list_all_upload_statuses()[0].get_counter()
 546         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 547         pub_num = h.list_all_publish_statuses()[0].get_counter()
 548         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 549         d = self.GET("/status", followRedirect=True)
 550         def _check(res):
 551             self.failUnless('Upload and Download Status' in res, res)
 552             self.failUnless('"down-%d"' % dl_num in res, res)
 553             self.failUnless('"up-%d"' % ul_num in res, res)
 554             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 555             self.failUnless('"publish-%d"' % pub_num in res, res)
 556             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 557         d.addCallback(_check)
 558         d.addCallback(lambda res: self.GET("/status/?t=json"))
 559         def _check_json(res):
 560             data = simplejson.loads(res)
 561             self.failUnless(isinstance(data, dict))
 562             #active = data["active"]
 563             # TODO: test more. We need a way to fake an active operation
 564             # here.
 565         d.addCallback(_check_json)
 566
 567         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 568         def _check_dl(res):
 569             self.failUnless("File Download Status" in res, res)
 570         d.addCallback(_check_dl)
 571         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 572         def _check_dl_json(res):
 573             data = simplejson.loads(res)
 574             self.failUnless(isinstance(data, dict))
 575             self.failUnless("read" in data)
 576             self.failUnlessEqual(data["read"][0]["length"], 120)
 577             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 578             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 579             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 580             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 581             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 582             # serverids[] keys are strings, since that's what JSON does, but
 583             # we'd really like them to be ints
 584             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 585             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 586             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 587             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 588             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 589             self.failUnless("dyhb" in data)
 590         d.addCallback(_check_dl_json)
 591         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 592         def _check_ul(res):
 593             self.failUnless("File Upload Status" in res, res)
 594         d.addCallback(_check_ul)
 595         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 596         def _check_mapupdate(res):
 597             self.failUnless("Mutable File Servermap Update Status" in res, res)
 598         d.addCallback(_check_mapupdate)
 599         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 600         def _check_publish(res):
 601             self.failUnless("Mutable File Publish Status" in res, res)
 602         d.addCallback(_check_publish)
 603         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 604         def _check_retrieve(res):
 605             self.failUnless("Mutable File Retrieve Status" in res, res)
 606         d.addCallback(_check_retrieve)
 607
 608         return d
 609
 610     def test_status_numbers(self):
 611         drrm = status.DownloadResultsRendererMixin()
 612         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 613         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 614         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 615         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 616         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 617         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 618         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 619         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 620         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 621
 622         urrm = status.UploadResultsRendererMixin()
 623         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 624         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 625         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 626         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 627         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 628         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 629         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 630         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 631         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 632
 633     def test_GET_FILEURL(self):
 634         d = self.GET(self.public_url + "/foo/bar.txt")
 635         d.addCallback(self.failUnlessIsBarDotTxt)
 636         return d
 637
 638     def test_GET_FILEURL_range(self):
 639         headers = {"range": "bytes=1-10"}
 640         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 641                      return_response=True)
 642         def _got((res, status, headers)):
 643             self.failUnlessReallyEqual(int(status), 206)
 644             self.failUnless(headers.has_key("content-range"))
 645             self.failUnlessReallyEqual(headers["content-range"][0],
 646                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 647             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 648         d.addCallback(_got)
 649         return d
 650
 651     def test_GET_FILEURL_partial_range(self):
 652         headers = {"range": "bytes=5-"}
 653         length  = len(self.BAR_CONTENTS)
 654         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 655                      return_response=True)
 656         def _got((res, status, headers)):
 657             self.failUnlessReallyEqual(int(status), 206)
 658             self.failUnless(headers.has_key("content-range"))
 659             self.failUnlessReallyEqual(headers["content-range"][0],
 660                                        "bytes 5-%d/%d" % (length-1, length))
 661             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 662         d.addCallback(_got)
 663         return d
 664
 665     def test_GET_FILEURL_partial_end_range(self):
 666         headers = {"range": "bytes=-5"}
 667         length  = len(self.BAR_CONTENTS)
 668         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 669                      return_response=True)
 670         def _got((res, status, headers)):
 671             self.failUnlessReallyEqual(int(status), 206)
 672             self.failUnless(headers.has_key("content-range"))
 673             self.failUnlessReallyEqual(headers["content-range"][0],
 674                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 675             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 676         d.addCallback(_got)
 677         return d
 678
 679     def test_GET_FILEURL_partial_range_overrun(self):
 680         headers = {"range": "bytes=100-200"}
 681         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 682                              "416 Requested Range not satisfiable",
 683                              "First beyond end of file",
 684                              self.GET, self.public_url + "/foo/bar.txt",
 685                              headers=headers)
 686         return d
 687
 688     def test_HEAD_FILEURL_range(self):
 689         headers = {"range": "bytes=1-10"}
 690         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 691                      return_response=True)
 692         def _got((res, status, headers)):
 693             self.failUnlessReallyEqual(res, "")
 694             self.failUnlessReallyEqual(int(status), 206)
 695             self.failUnless(headers.has_key("content-range"))
 696             self.failUnlessReallyEqual(headers["content-range"][0],
 697                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 698         d.addCallback(_got)
 699         return d
 700
 701     def test_HEAD_FILEURL_partial_range(self):
 702         headers = {"range": "bytes=5-"}
 703         length  = len(self.BAR_CONTENTS)
 704         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 705                      return_response=True)
 706         def _got((res, status, headers)):
 707             self.failUnlessReallyEqual(int(status), 206)
 708             self.failUnless(headers.has_key("content-range"))
 709             self.failUnlessReallyEqual(headers["content-range"][0],
 710                                        "bytes 5-%d/%d" % (length-1, length))
 711         d.addCallback(_got)
 712         return d
 713
 714     def test_HEAD_FILEURL_partial_end_range(self):
 715         headers = {"range": "bytes=-5"}
 716         length  = len(self.BAR_CONTENTS)
 717         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 718                      return_response=True)
 719         def _got((res, status, headers)):
 720             self.failUnlessReallyEqual(int(status), 206)
 721             self.failUnless(headers.has_key("content-range"))
 722             self.failUnlessReallyEqual(headers["content-range"][0],
 723                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 724         d.addCallback(_got)
 725         return d
 726
 727     def test_HEAD_FILEURL_partial_range_overrun(self):
 728         headers = {"range": "bytes=100-200"}
 729         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 730                              "416 Requested Range not satisfiable",
 731                              "",
 732                              self.HEAD, self.public_url + "/foo/bar.txt",
 733                              headers=headers)
 734         return d
 735
 736     def test_GET_FILEURL_range_bad(self):
 737         headers = {"range": "BOGUS=fizbop-quarnak"}
 738         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 739                      return_response=True)
 740         def _got((res, status, headers)):
 741             self.failUnlessReallyEqual(int(status), 200)
 742             self.failUnless(not headers.has_key("content-range"))
 743             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_HEAD_FILEURL(self):
 748         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 749         def _got((res, status, headers)):
 750             self.failUnlessReallyEqual(res, "")
 751             self.failUnlessReallyEqual(headers["content-length"][0],
 752                                        str(len(self.BAR_CONTENTS)))
 753             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 754         d.addCallback(_got)
 755         return d
 756
 757     def test_GET_FILEURL_named(self):
 758         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 759         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 760         d = self.GET(base + "/@@name=/blah.txt")
 761         d.addCallback(self.failUnlessIsBarDotTxt)
 762         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 763         d.addCallback(self.failUnlessIsBarDotTxt)
 764         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 765         d.addCallback(self.failUnlessIsBarDotTxt)
 766         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 767         d.addCallback(self.failUnlessIsBarDotTxt)
 768         save_url = base + "?save=true&filename=blah.txt"
 769         d.addCallback(lambda res: self.GET(save_url))
 770         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 771         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 772         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 773         u_url = base + "?save=true&filename=" + u_fn_e
 774         d.addCallback(lambda res: self.GET(u_url))
 775         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 776         return d
 777
 778     def test_PUT_FILEURL_named_bad(self):
 779         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 780         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 781                              "400 Bad Request",
 782                              "/file can only be used with GET or HEAD",
 783                              self.PUT, base + "/@@name=/blah.txt", "")
 784         return d
 785
 786     def test_GET_DIRURL_named_bad(self):
 787         base = "/file/%s" % urllib.quote(self._foo_uri)
 788         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 789                              "400 Bad Request",
 790                              "is not a file-cap",
 791                              self.GET, base + "/@@name=/blah.txt")
 792         return d
 793
 794     def test_GET_slash_file_bad(self):
 795         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 796                              "404 Not Found",
 797                              "/file must be followed by a file-cap and a name",
 798                              self.GET, "/file")
 799         return d
 800
 801     def test_GET_unhandled_URI_named(self):
 802         contents, n, newuri = self.makefile(12)
 803         verifier_cap = n.get_verify_cap().to_string()
 804         base = "/file/%s" % urllib.quote(verifier_cap)
 805         # client.create_node_from_uri() can't handle verify-caps
 806         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 807                              "400 Bad Request", "is not a file-cap",
 808                              self.GET, base)
 809         return d
 810
 811     def test_GET_unhandled_URI(self):
 812         contents, n, newuri = self.makefile(12)
 813         verifier_cap = n.get_verify_cap().to_string()
 814         base = "/uri/%s" % urllib.quote(verifier_cap)
 815         # client.create_node_from_uri() can't handle verify-caps
 816         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 817                              "400 Bad Request",
 818                              "GET unknown URI type: can only do t=info",
 819                              self.GET, base)
 820         return d
 821
 822     def test_GET_FILE_URI(self):
 823         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 824         d = self.GET(base)
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         return d
 827
 828     def test_GET_FILE_URI_badchild(self):
 829         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 830         errmsg = "Files have no children, certainly not named 'boguschild'"
 831         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 832                              "400 Bad Request", errmsg,
 833                              self.GET, base)
 834         return d
 835
 836     def test_PUT_FILE_URI_badchild(self):
 837         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 838         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 839         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 840                              "400 Bad Request", errmsg,
 841                              self.PUT, base, "")
 842         return d
 843
 844     # TODO: version of this with a Unicode filename
 845     def test_GET_FILEURL_save(self):
 846         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 847                      return_response=True)
 848         def _got((res, statuscode, headers)):
 849             content_disposition = headers["content-disposition"][0]
 850             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 851             self.failUnlessIsBarDotTxt(res)
 852         d.addCallback(_got)
 853         return d
 854
 855     def test_GET_FILEURL_missing(self):
 856         d = self.GET(self.public_url + "/foo/missing")
 857         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 858         return d
 859
 860     def test_PUT_overwrite_only_files(self):
 861         # create a directory, put a file in that directory.
 862         contents, n, filecap = self.makefile(8)
 863         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 864         d.addCallback(lambda res:
 865             self.PUT(self.public_url + "/foo/dir/file1.txt",
 866                      self.NEWFILE_CONTENTS))
 867         # try to overwrite the file with replace=only-files
 868         # (this should work)
 869         d.addCallback(lambda res:
 870             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 871                      filecap))
 872         d.addCallback(lambda res:
 873             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 874                  "There was already a child by that name, and you asked me "
 875                  "to not replace it",
 876                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 877                  filecap))
 878         return d
 879
 880     def test_PUT_NEWFILEURL(self):
 881         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 882         # TODO: we lose the response code, so we can't check this
 883         #self.failUnlessReallyEqual(responsecode, 201)
 884         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 885         d.addCallback(lambda res:
 886                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 887                                                       self.NEWFILE_CONTENTS))
 888         return d
 889
 890     def test_PUT_NEWFILEURL_not_mutable(self):
 891         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 892                      self.NEWFILE_CONTENTS)
 893         # TODO: we lose the response code, so we can't check this
 894         #self.failUnlessReallyEqual(responsecode, 201)
 895         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 896         d.addCallback(lambda res:
 897                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 898                                                       self.NEWFILE_CONTENTS))
 899         return d
 900
 901     def test_PUT_NEWFILEURL_range_bad(self):
 902         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 903         target = self.public_url + "/foo/new.txt"
 904         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 905                              "501 Not Implemented",
 906                              "Content-Range in PUT not yet supported",
 907                              # (and certainly not for immutable files)
 908                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 909                              headers=headers)
 910         d.addCallback(lambda res:
 911                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 912         return d
 913
 914     def test_PUT_NEWFILEURL_mutable(self):
 915         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 916                      self.NEWFILE_CONTENTS)
 917         # TODO: we lose the response code, so we can't check this
 918         #self.failUnlessReallyEqual(responsecode, 201)
 919         def _check_uri(res):
 920             u = uri.from_string_mutable_filenode(res)
 921             self.failUnless(u.is_mutable())
 922             self.failIf(u.is_readonly())
 923             return res
 924         d.addCallback(_check_uri)
 925         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 926         d.addCallback(lambda res:
 927                       self.failUnlessMutableChildContentsAre(self._foo_node,
 928                                                              u"new.txt",
 929                                                              self.NEWFILE_CONTENTS))
 930         return d
 931
 932     def test_PUT_NEWFILEURL_mutable_toobig(self):
 933         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 934                              "413 Request Entity Too Large",
 935                              "SDMF is limited to one segment, and 10001 > 10000",
 936                              self.PUT,
 937                              self.public_url + "/foo/new.txt?mutable=true",
 938                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 939         return d
 940
 941     def test_PUT_NEWFILEURL_replace(self):
 942         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 943         # TODO: we lose the response code, so we can't check this
 944         #self.failUnlessReallyEqual(responsecode, 200)
 945         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 946         d.addCallback(lambda res:
 947                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 948                                                       self.NEWFILE_CONTENTS))
 949         return d
 950
 951     def test_PUT_NEWFILEURL_bad_t(self):
 952         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 953                              "PUT to a file: bad t=bogus",
 954                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 955                              "contents")
 956         return d
 957
 958     def test_PUT_NEWFILEURL_no_replace(self):
 959         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 960                      self.NEWFILE_CONTENTS)
 961         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 962                   "409 Conflict",
 963                   "There was already a child by that name, and you asked me "
 964                   "to not replace it")
 965         return d
 966
 967     def test_PUT_NEWFILEURL_mkdirs(self):
 968         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 969         fn = self._foo_node
 970         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 971         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 972         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 973         d.addCallback(lambda res:
 974                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 975                                                       self.NEWFILE_CONTENTS))
 976         return d
 977
 978     def test_PUT_NEWFILEURL_blocked(self):
 979         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 980                      self.NEWFILE_CONTENTS)
 981         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 982                   "409 Conflict",
 983                   "Unable to create directory 'blockingfile': a file was in the way")
 984         return d
 985
 986     def test_PUT_NEWFILEURL_emptyname(self):
 987         # an empty pathname component (i.e. a double-slash) is disallowed
 988         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 989                              "400 Bad Request",
 990                              "The webapi does not allow empty pathname components",
 991                              self.PUT, self.public_url + "/foo//new.txt", "")
 992         return d
 993
 994     def test_DELETE_FILEURL(self):
 995         d = self.DELETE(self.public_url + "/foo/bar.txt")
 996         d.addCallback(lambda res:
 997                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 998         return d
 999
1000     def test_DELETE_FILEURL_missing(self):
1001         d = self.DELETE(self.public_url + "/foo/missing")
1002         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1003         return d
1004
1005     def test_DELETE_FILEURL_missing2(self):
1006         d = self.DELETE(self.public_url + "/missing/missing")
1007         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1008         return d
1009
1010     def failUnlessHasBarDotTxtMetadata(self, res):
1011         data = simplejson.loads(res)
1012         self.failUnless(isinstance(data, list))
1013         self.failUnlessIn("metadata", data[1])
1014         self.failUnlessIn("tahoe", data[1]["metadata"])
1015         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1016         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1017         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1018                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1019
1020     def test_GET_FILEURL_json(self):
1021         # twisted.web.http.parse_qs ignores any query args without an '=', so
1022         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1023         # instead. This may make it tricky to emulate the S3 interface
1024         # completely.
1025         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1026         def _check1(data):
1027             self.failUnlessIsBarJSON(data)
1028             self.failUnlessHasBarDotTxtMetadata(data)
1029             return
1030         d.addCallback(_check1)
1031         return d
1032
1033     def test_GET_FILEURL_json_missing(self):
1034         d = self.GET(self.public_url + "/foo/missing?json")
1035         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1036         return d
1037
1038     def test_GET_FILEURL_uri(self):
1039         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1040         def _check(res):
1041             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1042         d.addCallback(_check)
1043         d.addCallback(lambda res:
1044                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1045         def _check2(res):
1046             # for now, for files, uris and readonly-uris are the same
1047             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1048         d.addCallback(_check2)
1049         return d
1050
1051     def test_GET_FILEURL_badtype(self):
1052         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1053                                  "bad t=bogus",
1054                                  self.GET,
1055                                  self.public_url + "/foo/bar.txt?t=bogus")
1056         return d
1057
1058     def test_CSS_FILE(self):
1059         d = self.GET("/tahoe_css", followRedirect=True)
1060         def _check(res):
1061             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1062             self.failUnless(CSS_STYLE.search(res), res)
1063         d.addCallback(_check)
1064         return d
1065
1066     def test_GET_FILEURL_uri_missing(self):
1067         d = self.GET(self.public_url + "/foo/missing?t=uri")
1068         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1069         return d
1070
1071     def test_GET_DIRECTORY_html_banner(self):
1072         d = self.GET(self.public_url + "/foo", followRedirect=True)
1073         def _check(res):
1074             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1075         d.addCallback(_check)
1076         return d
1077
1078     def test_GET_DIRURL(self):
1079         # the addSlash means we get a redirect here
1080         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1081         ROOT = "../../.."
1082         d = self.GET(self.public_url + "/foo", followRedirect=True)
1083         def _check(res):
1084             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1085                             in res, res)
1086             # the FILE reference points to a URI, but it should end in bar.txt
1087             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1088                        (ROOT, urllib.quote(self._bar_txt_uri)))
1089             get_bar = "".join([r'<td>FILE</td>',
1090                                r'\s+<td>',
1091                                r'<a href="%s">bar.txt</a>' % bar_url,
1092                                r'</td>',
1093                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1094                                ])
1095             self.failUnless(re.search(get_bar, res), res)
1096             for line in res.split("\n"):
1097                 # find the line that contains the delete button for bar.txt
1098                 if ("form action" in line and
1099                     'value="delete"' in line and
1100                     'value="bar.txt"' in line):
1101                     # the form target should use a relative URL
1102                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1103                     self.failUnless(('action="%s"' % foo_url) in line, line)
1104                     # and the when_done= should too
1105                     #done_url = urllib.quote(???)
1106                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1107                     #                in line, line)
1108                     break
1109             else:
1110                 self.fail("unable to find delete-bar.txt line", res)
1111
1112             # the DIR reference just points to a URI
1113             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1114             get_sub = ((r'<td>DIR</td>')
1115                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1116             self.failUnless(re.search(get_sub, res), res)
1117         d.addCallback(_check)
1118
1119         # look at a readonly directory
1120         d.addCallback(lambda res:
1121                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1122         def _check2(res):
1123             self.failUnless("(read-only)" in res, res)
1124             self.failIf("Upload a file" in res, res)
1125         d.addCallback(_check2)
1126
1127         # and at a directory that contains a readonly directory
1128         d.addCallback(lambda res:
1129                       self.GET(self.public_url, followRedirect=True))
1130         def _check3(res):
1131             self.failUnless(re.search('<td>DIR-RO</td>'
1132                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1133         d.addCallback(_check3)
1134
1135         # and an empty directory
1136         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1137         def _check4(res):
1138             self.failUnless("directory is empty" in res, res)
1139             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1140             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1141         d.addCallback(_check4)
1142
1143         # and at a literal directory
1144         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1145         d.addCallback(lambda res:
1146                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1147         def _check5(res):
1148             self.failUnless('(immutable)' in res, res)
1149             self.failUnless(re.search('<td>FILE</td>'
1150                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1151         d.addCallback(_check5)
1152         return d
1153
1154     def test_GET_DIRURL_badtype(self):
1155         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1156                                  400, "Bad Request",
1157                                  "bad t=bogus",
1158                                  self.GET,
1159                                  self.public_url + "/foo?t=bogus")
1160         return d
1161
1162     def test_GET_DIRURL_json(self):
1163         d = self.GET(self.public_url + "/foo?t=json")
1164         d.addCallback(self.failUnlessIsFooJSON)
1165         return d
1166
1167
1168     def test_POST_DIRURL_manifest_no_ophandle(self):
1169         d = self.shouldFail2(error.Error,
1170                              "test_POST_DIRURL_manifest_no_ophandle",
1171                              "400 Bad Request",
1172                              "slow operation requires ophandle=",
1173                              self.POST, self.public_url, t="start-manifest")
1174         return d
1175
1176     def test_POST_DIRURL_manifest(self):
1177         d = defer.succeed(None)
1178         def getman(ignored, output):
1179             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1180                           followRedirect=True)
1181             d.addCallback(self.wait_for_operation, "125")
1182             d.addCallback(self.get_operation_results, "125", output)
1183             return d
1184         d.addCallback(getman, None)
1185         def _got_html(manifest):
1186             self.failUnless("Manifest of SI=" in manifest)
1187             self.failUnless("<td>sub</td>" in manifest)
1188             self.failUnless(self._sub_uri in manifest)
1189             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1190         d.addCallback(_got_html)
1191
1192         # both t=status and unadorned GET should be identical
1193         d.addCallback(lambda res: self.GET("/operations/125"))
1194         d.addCallback(_got_html)
1195
1196         d.addCallback(getman, "html")
1197         d.addCallback(_got_html)
1198         d.addCallback(getman, "text")
1199         def _got_text(manifest):
1200             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1201             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1202         d.addCallback(_got_text)
1203         d.addCallback(getman, "JSON")
1204         def _got_json(res):
1205             data = res["manifest"]
1206             got = {}
1207             for (path_list, cap) in data:
1208                 got[tuple(path_list)] = cap
1209             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1210             self.failUnless((u"sub",u"baz.txt") in got)
1211             self.failUnless("finished" in res)
1212             self.failUnless("origin" in res)
1213             self.failUnless("storage-index" in res)
1214             self.failUnless("verifycaps" in res)
1215             self.failUnless("stats" in res)
1216         d.addCallback(_got_json)
1217         return d
1218
1219     def test_POST_DIRURL_deepsize_no_ophandle(self):
1220         d = self.shouldFail2(error.Error,
1221                              "test_POST_DIRURL_deepsize_no_ophandle",
1222                              "400 Bad Request",
1223                              "slow operation requires ophandle=",
1224                              self.POST, self.public_url, t="start-deep-size")
1225         return d
1226
1227     def test_POST_DIRURL_deepsize(self):
1228         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1229                       followRedirect=True)
1230         d.addCallback(self.wait_for_operation, "126")
1231         d.addCallback(self.get_operation_results, "126", "json")
1232         def _got_json(data):
1233             self.failUnlessReallyEqual(data["finished"], True)
1234             size = data["size"]
1235             self.failUnless(size > 1000)
1236         d.addCallback(_got_json)
1237         d.addCallback(self.get_operation_results, "126", "text")
1238         def _got_text(res):
1239             mo = re.search(r'^size: (\d+)$', res, re.M)
1240             self.failUnless(mo, res)
1241             size = int(mo.group(1))
1242             # with directories, the size varies.
1243             self.failUnless(size > 1000)
1244         d.addCallback(_got_text)
1245         return d
1246
1247     def test_POST_DIRURL_deepstats_no_ophandle(self):
1248         d = self.shouldFail2(error.Error,
1249                              "test_POST_DIRURL_deepstats_no_ophandle",
1250                              "400 Bad Request",
1251                              "slow operation requires ophandle=",
1252                              self.POST, self.public_url, t="start-deep-stats")
1253         return d
1254
1255     def test_POST_DIRURL_deepstats(self):
1256         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1257                       followRedirect=True)
1258         d.addCallback(self.wait_for_operation, "127")
1259         d.addCallback(self.get_operation_results, "127", "json")
1260         def _got_json(stats):
1261             expected = {"count-immutable-files": 3,
1262                         "count-mutable-files": 0,
1263                         "count-literal-files": 0,
1264                         "count-files": 3,
1265                         "count-directories": 3,
1266                         "size-immutable-files": 57,
1267                         "size-literal-files": 0,
1268                         #"size-directories": 1912, # varies
1269                         #"largest-directory": 1590,
1270                         "largest-directory-children": 5,
1271                         "largest-immutable-file": 19,
1272                         }
1273             for k,v in expected.iteritems():
1274                 self.failUnlessReallyEqual(stats[k], v,
1275                                            "stats[%s] was %s, not %s" %
1276                                            (k, stats[k], v))
1277             self.failUnlessReallyEqual(stats["size-files-histogram"],
1278                                        [ [11, 31, 3] ])
1279         d.addCallback(_got_json)
1280         return d
1281
1282     def test_POST_DIRURL_stream_manifest(self):
1283         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1284         def _check(res):
1285             self.failUnless(res.endswith("\n"))
1286             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1287             self.failUnlessReallyEqual(len(units), 7)
1288             self.failUnlessEqual(units[-1]["type"], "stats")
1289             first = units[0]
1290             self.failUnlessEqual(first["path"], [])
1291             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1292             self.failUnlessEqual(first["type"], "directory")
1293             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1294             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1295             self.failIfEqual(baz["storage-index"], None)
1296             self.failIfEqual(baz["verifycap"], None)
1297             self.failIfEqual(baz["repaircap"], None)
1298             return
1299         d.addCallback(_check)
1300         return d
1301
1302     def test_GET_DIRURL_uri(self):
1303         d = self.GET(self.public_url + "/foo?t=uri")
1304         def _check(res):
1305             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1306         d.addCallback(_check)
1307         return d
1308
1309     def test_GET_DIRURL_readonly_uri(self):
1310         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1311         def _check(res):
1312             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1313         d.addCallback(_check)
1314         return d
1315
1316     def test_PUT_NEWDIRURL(self):
1317         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1318         d.addCallback(lambda res:
1319                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1320         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1321         d.addCallback(self.failUnlessNodeKeysAre, [])
1322         return d
1323
1324     def test_POST_NEWDIRURL(self):
1325         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1326         d.addCallback(lambda res:
1327                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1328         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1329         d.addCallback(self.failUnlessNodeKeysAre, [])
1330         return d
1331
1332     def test_POST_NEWDIRURL_emptyname(self):
1333         # an empty pathname component (i.e. a double-slash) is disallowed
1334         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1335                              "400 Bad Request",
1336                              "The webapi does not allow empty pathname components, i.e. a double slash",
1337                              self.POST, self.public_url + "//?t=mkdir")
1338         return d
1339
1340     def test_POST_NEWDIRURL_initial_children(self):
1341         (newkids, caps) = self._create_initial_children()
1342         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1343                        simplejson.dumps(newkids))
1344         def _check(uri):
1345             n = self.s.create_node_from_uri(uri.strip())
1346             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1347             d2.addCallback(lambda ign:
1348                            self.failUnlessROChildURIIs(n, u"child-imm",
1349                                                        caps['filecap1']))
1350             d2.addCallback(lambda ign:
1351                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1352                                                        caps['filecap2']))
1353             d2.addCallback(lambda ign:
1354                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1355                                                        caps['filecap3']))
1356             d2.addCallback(lambda ign:
1357                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1358                                                        caps['unknown_rocap']))
1359             d2.addCallback(lambda ign:
1360                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1361                                                        caps['unknown_rwcap']))
1362             d2.addCallback(lambda ign:
1363                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1364                                                        caps['unknown_immcap']))
1365             d2.addCallback(lambda ign:
1366                            self.failUnlessRWChildURIIs(n, u"dirchild",
1367                                                        caps['dircap']))
1368             d2.addCallback(lambda ign:
1369                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1370                                                        caps['litdircap']))
1371             d2.addCallback(lambda ign:
1372                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1373                                                        caps['emptydircap']))
1374             return d2
1375         d.addCallback(_check)
1376         d.addCallback(lambda res:
1377                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1378         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1379         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1380         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1381         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1382         return d
1383
1384     def test_POST_NEWDIRURL_immutable(self):
1385         (newkids, caps) = self._create_immutable_children()
1386         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1387                        simplejson.dumps(newkids))
1388         def _check(uri):
1389             n = self.s.create_node_from_uri(uri.strip())
1390             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1391             d2.addCallback(lambda ign:
1392                            self.failUnlessROChildURIIs(n, u"child-imm",
1393                                                        caps['filecap1']))
1394             d2.addCallback(lambda ign:
1395                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1396                                                        caps['unknown_immcap']))
1397             d2.addCallback(lambda ign:
1398                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1399                                                        caps['immdircap']))
1400             d2.addCallback(lambda ign:
1401                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1402                                                        caps['litdircap']))
1403             d2.addCallback(lambda ign:
1404                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1405                                                        caps['emptydircap']))
1406             return d2
1407         d.addCallback(_check)
1408         d.addCallback(lambda res:
1409                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1410         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1411         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1412         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1413         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1414         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1415         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1416         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1417         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1418         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1419         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1420         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1421         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1422         d.addErrback(self.explain_web_error)
1423         return d
1424
1425     def test_POST_NEWDIRURL_immutable_bad(self):
1426         (newkids, caps) = self._create_initial_children()
1427         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1428                              "400 Bad Request",
1429                              "needed to be immutable but was not",
1430                              self.POST2,
1431                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1432                              simplejson.dumps(newkids))
1433         return d
1434
1435     def test_PUT_NEWDIRURL_exists(self):
1436         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1437         d.addCallback(lambda res:
1438                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1439         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1440         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1441         return d
1442
1443     def test_PUT_NEWDIRURL_blocked(self):
1444         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1445                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1446                              self.PUT,
1447                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1448         d.addCallback(lambda res:
1449                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1450         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1451         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1452         return d
1453
1454     def test_PUT_NEWDIRURL_mkdir_p(self):
1455         d = defer.succeed(None)
1456         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1457         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1458         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1459         def mkdir_p(mkpnode):
1460             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1461             d = self.POST(url)
1462             def made_subsub(ssuri):
1463                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1464                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1465                 d = self.POST(url)
1466                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1467                 return d
1468             d.addCallback(made_subsub)
1469             return d
1470         d.addCallback(mkdir_p)
1471         return d
1472
1473     def test_PUT_NEWDIRURL_mkdirs(self):
1474         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1475         d.addCallback(lambda res:
1476                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1477         d.addCallback(lambda res:
1478                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1479         d.addCallback(lambda res:
1480                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1481         d.addCallback(self.failUnlessNodeKeysAre, [])
1482         return d
1483
1484     def test_DELETE_DIRURL(self):
1485         d = self.DELETE(self.public_url + "/foo")
1486         d.addCallback(lambda res:
1487                       self.failIfNodeHasChild(self.public_root, u"foo"))
1488         return d
1489
1490     def test_DELETE_DIRURL_missing(self):
1491         d = self.DELETE(self.public_url + "/foo/missing")
1492         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1493         d.addCallback(lambda res:
1494                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1495         return d
1496
1497     def test_DELETE_DIRURL_missing2(self):
1498         d = self.DELETE(self.public_url + "/missing")
1499         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1500         return d
1501
1502     def dump_root(self):
1503         print "NODEWALK"
1504         w = webish.DirnodeWalkerMixin()
1505         def visitor(childpath, childnode, metadata):
1506             print childpath
1507         d = w.walk(self.public_root, visitor)
1508         return d
1509
1510     def failUnlessNodeKeysAre(self, node, expected_keys):
1511         for k in expected_keys:
1512             assert isinstance(k, unicode)
1513         d = node.list()
1514         def _check(children):
1515             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1516         d.addCallback(_check)
1517         return d
1518     def failUnlessNodeHasChild(self, node, name):
1519         assert isinstance(name, unicode)
1520         d = node.list()
1521         def _check(children):
1522             self.failUnless(name in children)
1523         d.addCallback(_check)
1524         return d
1525     def failIfNodeHasChild(self, node, name):
1526         assert isinstance(name, unicode)
1527         d = node.list()
1528         def _check(children):
1529             self.failIf(name in children)
1530         d.addCallback(_check)
1531         return d
1532
1533     def failUnlessChildContentsAre(self, node, name, expected_contents):
1534         assert isinstance(name, unicode)
1535         d = node.get_child_at_path(name)
1536         d.addCallback(lambda node: download_to_data(node))
1537         def _check(contents):
1538             self.failUnlessReallyEqual(contents, expected_contents)
1539         d.addCallback(_check)
1540         return d
1541
1542     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1543         assert isinstance(name, unicode)
1544         d = node.get_child_at_path(name)
1545         d.addCallback(lambda node: node.download_best_version())
1546         def _check(contents):
1547             self.failUnlessReallyEqual(contents, expected_contents)
1548         d.addCallback(_check)
1549         return d
1550
1551     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1552         assert isinstance(name, unicode)
1553         d = node.get_child_at_path(name)
1554         def _check(child):
1555             self.failUnless(child.is_unknown() or not child.is_readonly())
1556             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1557             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1558             expected_ro_uri = self._make_readonly(expected_uri)
1559             if expected_ro_uri:
1560                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1561         d.addCallback(_check)
1562         return d
1563
1564     def failUnlessROChildURIIs(self, node, name, expected_uri):
1565         assert isinstance(name, unicode)
1566         d = node.get_child_at_path(name)
1567         def _check(child):
1568             self.failUnless(child.is_unknown() or child.is_readonly())
1569             self.failUnlessReallyEqual(child.get_write_uri(), None)
1570             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1571             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1572         d.addCallback(_check)
1573         return d
1574
1575     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1576         assert isinstance(name, unicode)
1577         d = node.get_child_at_path(name)
1578         def _check(child):
1579             self.failUnless(child.is_unknown() or not child.is_readonly())
1580             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1581             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1582             expected_ro_uri = self._make_readonly(got_uri)
1583             if expected_ro_uri:
1584                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1585         d.addCallback(_check)
1586         return d
1587
1588     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1589         assert isinstance(name, unicode)
1590         d = node.get_child_at_path(name)
1591         def _check(child):
1592             self.failUnless(child.is_unknown() or child.is_readonly())
1593             self.failUnlessReallyEqual(child.get_write_uri(), None)
1594             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1595             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1596         d.addCallback(_check)
1597         return d
1598
1599     def failUnlessCHKURIHasContents(self, got_uri, contents):
1600         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1601
1602     def test_POST_upload(self):
1603         d = self.POST(self.public_url + "/foo", t="upload",
1604                       file=("new.txt", self.NEWFILE_CONTENTS))
1605         fn = self._foo_node
1606         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1607         d.addCallback(lambda res:
1608                       self.failUnlessChildContentsAre(fn, u"new.txt",
1609                                                       self.NEWFILE_CONTENTS))
1610         return d
1611
1612     def test_POST_upload_unicode(self):
1613         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1614         d = self.POST(self.public_url + "/foo", t="upload",
1615                       file=(filename, self.NEWFILE_CONTENTS))
1616         fn = self._foo_node
1617         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1618         d.addCallback(lambda res:
1619                       self.failUnlessChildContentsAre(fn, filename,
1620                                                       self.NEWFILE_CONTENTS))
1621         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1622         d.addCallback(lambda res: self.GET(target_url))
1623         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1624                                                                   self.NEWFILE_CONTENTS,
1625                                                                   contents))
1626         return d
1627
1628     def test_POST_upload_unicode_named(self):
1629         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1630         d = self.POST(self.public_url + "/foo", t="upload",
1631                       name=filename,
1632                       file=("overridden", self.NEWFILE_CONTENTS))
1633         fn = self._foo_node
1634         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1635         d.addCallback(lambda res:
1636                       self.failUnlessChildContentsAre(fn, filename,
1637                                                       self.NEWFILE_CONTENTS))
1638         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1639         d.addCallback(lambda res: self.GET(target_url))
1640         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1641                                                                   self.NEWFILE_CONTENTS,
1642                                                                   contents))
1643         return d
1644
1645     def test_POST_upload_no_link(self):
1646         d = self.POST("/uri", t="upload",
1647                       file=("new.txt", self.NEWFILE_CONTENTS))
1648         def _check_upload_results(page):
1649             # this should be a page which describes the results of the upload
1650             # that just finished.
1651             self.failUnless("Upload Results:" in page)
1652             self.failUnless("URI:" in page)
1653             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1654             mo = uri_re.search(page)
1655             self.failUnless(mo, page)
1656             new_uri = mo.group(1)
1657             return new_uri
1658         d.addCallback(_check_upload_results)
1659         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1660         return d
1661
1662     def test_POST_upload_no_link_whendone(self):
1663         d = self.POST("/uri", t="upload", when_done="/",
1664                       file=("new.txt", self.NEWFILE_CONTENTS))
1665         d.addBoth(self.shouldRedirect, "/")
1666         return d
1667
1668     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1669         d = defer.maybeDeferred(callable, *args, **kwargs)
1670         def done(res):
1671             if isinstance(res, failure.Failure):
1672                 res.trap(error.PageRedirect)
1673                 statuscode = res.value.status
1674                 target = res.value.location
1675                 return checker(statuscode, target)
1676             self.fail("%s: callable was supposed to redirect, not return '%s'"
1677                       % (which, res))
1678         d.addBoth(done)
1679         return d
1680
1681     def test_POST_upload_no_link_whendone_results(self):
1682         def check(statuscode, target):
1683             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1684             self.failUnless(target.startswith(self.webish_url), target)
1685             return client.getPage(target, method="GET")
1686         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1687                                  check,
1688                                  self.POST, "/uri", t="upload",
1689                                  when_done="/uri/%(uri)s",
1690                                  file=("new.txt", self.NEWFILE_CONTENTS))
1691         d.addCallback(lambda res:
1692                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1693         return d
1694
1695     def test_POST_upload_no_link_mutable(self):
1696         d = self.POST("/uri", t="upload", mutable="true",
1697                       file=("new.txt", self.NEWFILE_CONTENTS))
1698         def _check(filecap):
1699             filecap = filecap.strip()
1700             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1701             self.filecap = filecap
1702             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1703             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1704             n = self.s.create_node_from_uri(filecap)
1705             return n.download_best_version()
1706         d.addCallback(_check)
1707         def _check2(data):
1708             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1709             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1710         d.addCallback(_check2)
1711         def _check3(data):
1712             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1713             return self.GET("/file/%s" % urllib.quote(self.filecap))
1714         d.addCallback(_check3)
1715         def _check4(data):
1716             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1717         d.addCallback(_check4)
1718         return d
1719
1720     def test_POST_upload_no_link_mutable_toobig(self):
1721         d = self.shouldFail2(error.Error,
1722                              "test_POST_upload_no_link_mutable_toobig",
1723                              "413 Request Entity Too Large",
1724                              "SDMF is limited to one segment, and 10001 > 10000",
1725                              self.POST,
1726                              "/uri", t="upload", mutable="true",
1727                              file=("new.txt",
1728                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1729         return d
1730
1731     def test_POST_upload_mutable(self):
1732         # this creates a mutable file
1733         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1734                       file=("new.txt", self.NEWFILE_CONTENTS))
1735         fn = self._foo_node
1736         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1737         d.addCallback(lambda res:
1738                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1739                                                              self.NEWFILE_CONTENTS))
1740         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1741         def _got(newnode):
1742             self.failUnless(IMutableFileNode.providedBy(newnode))
1743             self.failUnless(newnode.is_mutable())
1744             self.failIf(newnode.is_readonly())
1745             self._mutable_node = newnode
1746             self._mutable_uri = newnode.get_uri()
1747         d.addCallback(_got)
1748
1749         # now upload it again and make sure that the URI doesn't change
1750         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1751         d.addCallback(lambda res:
1752                       self.POST(self.public_url + "/foo", t="upload",
1753                                 mutable="true",
1754                                 file=("new.txt", NEWER_CONTENTS)))
1755         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1756         d.addCallback(lambda res:
1757                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1758                                                              NEWER_CONTENTS))
1759         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1760         def _got2(newnode):
1761             self.failUnless(IMutableFileNode.providedBy(newnode))
1762             self.failUnless(newnode.is_mutable())
1763             self.failIf(newnode.is_readonly())
1764             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1765         d.addCallback(_got2)
1766
1767         # upload a second time, using PUT instead of POST
1768         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1769         d.addCallback(lambda res:
1770                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1771         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1772         d.addCallback(lambda res:
1773                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1774                                                              NEW2_CONTENTS))
1775
1776         # finally list the directory, since mutable files are displayed
1777         # slightly differently
1778
1779         d.addCallback(lambda res:
1780                       self.GET(self.public_url + "/foo/",
1781                                followRedirect=True))
1782         def _check_page(res):
1783             # TODO: assert more about the contents
1784             self.failUnless("SSK" in res)
1785             return res
1786         d.addCallback(_check_page)
1787
1788         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1789         def _got3(newnode):
1790             self.failUnless(IMutableFileNode.providedBy(newnode))
1791             self.failUnless(newnode.is_mutable())
1792             self.failIf(newnode.is_readonly())
1793             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1794         d.addCallback(_got3)
1795
1796         # look at the JSON form of the enclosing directory
1797         d.addCallback(lambda res:
1798                       self.GET(self.public_url + "/foo/?t=json",
1799                                followRedirect=True))
1800         def _check_page_json(res):
1801             parsed = simplejson.loads(res)
1802             self.failUnlessEqual(parsed[0], "dirnode")
1803             children = dict( [(unicode(name),value)
1804                               for (name,value)
1805                               in parsed[1]["children"].iteritems()] )
1806             self.failUnless(u"new.txt" in children)
1807             new_json = children[u"new.txt"]
1808             self.failUnlessEqual(new_json[0], "filenode")
1809             self.failUnless(new_json[1]["mutable"])
1810             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
1811             ro_uri = self._mutable_node.get_readonly().to_string()
1812             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
1813         d.addCallback(_check_page_json)
1814
1815         # and the JSON form of the file
1816         d.addCallback(lambda res:
1817                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1818         def _check_file_json(res):
1819             parsed = simplejson.loads(res)
1820             self.failUnlessEqual(parsed[0], "filenode")
1821             self.failUnless(parsed[1]["mutable"])
1822             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
1823             ro_uri = self._mutable_node.get_readonly().to_string()
1824             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
1825         d.addCallback(_check_file_json)
1826
1827         # and look at t=uri and t=readonly-uri
1828         d.addCallback(lambda res:
1829                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1830         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1831         d.addCallback(lambda res:
1832                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1833         def _check_ro_uri(res):
1834             ro_uri = self._mutable_node.get_readonly().to_string()
1835             self.failUnlessReallyEqual(res, ro_uri)
1836         d.addCallback(_check_ro_uri)
1837
1838         # make sure we can get to it from /uri/URI
1839         d.addCallback(lambda res:
1840                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1841         d.addCallback(lambda res:
1842                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1843
1844         # and that HEAD computes the size correctly
1845         d.addCallback(lambda res:
1846                       self.HEAD(self.public_url + "/foo/new.txt",
1847                                 return_response=True))
1848         def _got_headers((res, status, headers)):
1849             self.failUnlessReallyEqual(res, "")
1850             self.failUnlessReallyEqual(headers["content-length"][0],
1851                                        str(len(NEW2_CONTENTS)))
1852             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1853         d.addCallback(_got_headers)
1854
1855         # make sure that size errors are displayed correctly for overwrite
1856         d.addCallback(lambda res:
1857                       self.shouldFail2(error.Error,
1858                                        "test_POST_upload_mutable-toobig",
1859                                        "413 Request Entity Too Large",
1860                                        "SDMF is limited to one segment, and 10001 > 10000",
1861                                        self.POST,
1862                                        self.public_url + "/foo", t="upload",
1863                                        mutable="true",
1864                                        file=("new.txt",
1865                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1866                                        ))
1867
1868         d.addErrback(self.dump_error)
1869         return d
1870
1871     def test_POST_upload_mutable_toobig(self):
1872         d = self.shouldFail2(error.Error,
1873                              "test_POST_upload_mutable_toobig",
1874                              "413 Request Entity Too Large",
1875                              "SDMF is limited to one segment, and 10001 > 10000",
1876                              self.POST,
1877                              self.public_url + "/foo",
1878                              t="upload", mutable="true",
1879                              file=("new.txt",
1880                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1881         return d
1882
1883     def dump_error(self, f):
1884         # if the web server returns an error code (like 400 Bad Request),
1885         # web.client.getPage puts the HTTP response body into the .response
1886         # attribute of the exception object that it gives back. It does not
1887         # appear in the Failure's repr(), so the ERROR that trial displays
1888         # will be rather terse and unhelpful. addErrback this method to the
1889         # end of your chain to get more information out of these errors.
1890         if f.check(error.Error):
1891             print "web.error.Error:"
1892             print f
1893             print f.value.response
1894         return f
1895
1896     def test_POST_upload_replace(self):
1897         d = self.POST(self.public_url + "/foo", t="upload",
1898                       file=("bar.txt", self.NEWFILE_CONTENTS))
1899         fn = self._foo_node
1900         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1901         d.addCallback(lambda res:
1902                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1903                                                       self.NEWFILE_CONTENTS))
1904         return d
1905
1906     def test_POST_upload_no_replace_ok(self):
1907         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1908                       file=("new.txt", self.NEWFILE_CONTENTS))
1909         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1910         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1911                                                              self.NEWFILE_CONTENTS))
1912         return d
1913
1914     def test_POST_upload_no_replace_queryarg(self):
1915         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1916                       file=("bar.txt", self.NEWFILE_CONTENTS))
1917         d.addBoth(self.shouldFail, error.Error,
1918                   "POST_upload_no_replace_queryarg",
1919                   "409 Conflict",
1920                   "There was already a child by that name, and you asked me "
1921                   "to not replace it")
1922         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1923         d.addCallback(self.failUnlessIsBarDotTxt)
1924         return d
1925
1926     def test_POST_upload_no_replace_field(self):
1927         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1928                       file=("bar.txt", self.NEWFILE_CONTENTS))
1929         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1930                   "409 Conflict",
1931                   "There was already a child by that name, and you asked me "
1932                   "to not replace it")
1933         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1934         d.addCallback(self.failUnlessIsBarDotTxt)
1935         return d
1936
1937     def test_POST_upload_whendone(self):
1938         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1939                       file=("new.txt", self.NEWFILE_CONTENTS))
1940         d.addBoth(self.shouldRedirect, "/THERE")
1941         fn = self._foo_node
1942         d.addCallback(lambda res:
1943                       self.failUnlessChildContentsAre(fn, u"new.txt",
1944                                                       self.NEWFILE_CONTENTS))
1945         return d
1946
1947     def test_POST_upload_named(self):
1948         fn = self._foo_node
1949         d = self.POST(self.public_url + "/foo", t="upload",
1950                       name="new.txt", file=self.NEWFILE_CONTENTS)
1951         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1952         d.addCallback(lambda res:
1953                       self.failUnlessChildContentsAre(fn, u"new.txt",
1954                                                       self.NEWFILE_CONTENTS))
1955         return d
1956
1957     def test_POST_upload_named_badfilename(self):
1958         d = self.POST(self.public_url + "/foo", t="upload",
1959                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1960         d.addBoth(self.shouldFail, error.Error,
1961                   "test_POST_upload_named_badfilename",
1962                   "400 Bad Request",
1963                   "name= may not contain a slash",
1964                   )
1965         # make sure that nothing was added
1966         d.addCallback(lambda res:
1967                       self.failUnlessNodeKeysAre(self._foo_node,
1968                                                  [u"bar.txt", u"blockingfile",
1969                                                   u"empty", u"n\u00fc.txt",
1970                                                   u"sub"]))
1971         return d
1972
1973     def test_POST_FILEURL_check(self):
1974         bar_url = self.public_url + "/foo/bar.txt"
1975         d = self.POST(bar_url, t="check")
1976         def _check(res):
1977             self.failUnless("Healthy :" in res)
1978         d.addCallback(_check)
1979         redir_url = "http://allmydata.org/TARGET"
1980         def _check2(statuscode, target):
1981             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1982             self.failUnlessReallyEqual(target, redir_url)
1983         d.addCallback(lambda res:
1984                       self.shouldRedirect2("test_POST_FILEURL_check",
1985                                            _check2,
1986                                            self.POST, bar_url,
1987                                            t="check",
1988                                            when_done=redir_url))
1989         d.addCallback(lambda res:
1990                       self.POST(bar_url, t="check", return_to=redir_url))
1991         def _check3(res):
1992             self.failUnless("Healthy :" in res)
1993             self.failUnless("Return to file" in res)
1994             self.failUnless(redir_url in res)
1995         d.addCallback(_check3)
1996
1997         d.addCallback(lambda res:
1998                       self.POST(bar_url, t="check", output="JSON"))
1999         def _check_json(res):
2000             data = simplejson.loads(res)
2001             self.failUnless("storage-index" in data)
2002             self.failUnless(data["results"]["healthy"])
2003         d.addCallback(_check_json)
2004
2005         return d
2006
2007     def test_POST_FILEURL_check_and_repair(self):
2008         bar_url = self.public_url + "/foo/bar.txt"
2009         d = self.POST(bar_url, t="check", repair="true")
2010         def _check(res):
2011             self.failUnless("Healthy :" in res)
2012         d.addCallback(_check)
2013         redir_url = "http://allmydata.org/TARGET"
2014         def _check2(statuscode, target):
2015             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2016             self.failUnlessReallyEqual(target, redir_url)
2017         d.addCallback(lambda res:
2018                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2019                                            _check2,
2020                                            self.POST, bar_url,
2021                                            t="check", repair="true",
2022                                            when_done=redir_url))
2023         d.addCallback(lambda res:
2024                       self.POST(bar_url, t="check", return_to=redir_url))
2025         def _check3(res):
2026             self.failUnless("Healthy :" in res)
2027             self.failUnless("Return to file" in res)
2028             self.failUnless(redir_url in res)
2029         d.addCallback(_check3)
2030         return d
2031
2032     def test_POST_DIRURL_check(self):
2033         foo_url = self.public_url + "/foo/"
2034         d = self.POST(foo_url, t="check")
2035         def _check(res):
2036             self.failUnless("Healthy :" in res, res)
2037         d.addCallback(_check)
2038         redir_url = "http://allmydata.org/TARGET"
2039         def _check2(statuscode, target):
2040             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2041             self.failUnlessReallyEqual(target, redir_url)
2042         d.addCallback(lambda res:
2043                       self.shouldRedirect2("test_POST_DIRURL_check",
2044                                            _check2,
2045                                            self.POST, foo_url,
2046                                            t="check",
2047                                            when_done=redir_url))
2048         d.addCallback(lambda res:
2049                       self.POST(foo_url, t="check", return_to=redir_url))
2050         def _check3(res):
2051             self.failUnless("Healthy :" in res, res)
2052             self.failUnless("Return to file/directory" in res)
2053             self.failUnless(redir_url in res)
2054         d.addCallback(_check3)
2055
2056         d.addCallback(lambda res:
2057                       self.POST(foo_url, t="check", output="JSON"))
2058         def _check_json(res):
2059             data = simplejson.loads(res)
2060             self.failUnless("storage-index" in data)
2061             self.failUnless(data["results"]["healthy"])
2062         d.addCallback(_check_json)
2063
2064         return d
2065
2066     def test_POST_DIRURL_check_and_repair(self):
2067         foo_url = self.public_url + "/foo/"
2068         d = self.POST(foo_url, t="check", repair="true")
2069         def _check(res):
2070             self.failUnless("Healthy :" in res, res)
2071         d.addCallback(_check)
2072         redir_url = "http://allmydata.org/TARGET"
2073         def _check2(statuscode, target):
2074             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2075             self.failUnlessReallyEqual(target, redir_url)
2076         d.addCallback(lambda res:
2077                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2078                                            _check2,
2079                                            self.POST, foo_url,
2080                                            t="check", repair="true",
2081                                            when_done=redir_url))
2082         d.addCallback(lambda res:
2083                       self.POST(foo_url, t="check", return_to=redir_url))
2084         def _check3(res):
2085             self.failUnless("Healthy :" in res)
2086             self.failUnless("Return to file/directory" in res)
2087             self.failUnless(redir_url in res)
2088         d.addCallback(_check3)
2089         return d
2090
2091     def wait_for_operation(self, ignored, ophandle):
2092         url = "/operations/" + ophandle
2093         url += "?t=status&output=JSON"
2094         d = self.GET(url)
2095         def _got(res):
2096             data = simplejson.loads(res)
2097             if not data["finished"]:
2098                 d = self.stall(delay=1.0)
2099                 d.addCallback(self.wait_for_operation, ophandle)
2100                 return d
2101             return data
2102         d.addCallback(_got)
2103         return d
2104
2105     def get_operation_results(self, ignored, ophandle, output=None):
2106         url = "/operations/" + ophandle
2107         url += "?t=status"
2108         if output:
2109             url += "&output=" + output
2110         d = self.GET(url)
2111         def _got(res):
2112             if output and output.lower() == "json":
2113                 return simplejson.loads(res)
2114             return res
2115         d.addCallback(_got)
2116         return d
2117
2118     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2119         d = self.shouldFail2(error.Error,
2120                              "test_POST_DIRURL_deepcheck_no_ophandle",
2121                              "400 Bad Request",
2122                              "slow operation requires ophandle=",
2123                              self.POST, self.public_url, t="start-deep-check")
2124         return d
2125
2126     def test_POST_DIRURL_deepcheck(self):
2127         def _check_redirect(statuscode, target):
2128             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2129             self.failUnless(target.endswith("/operations/123"))
2130         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2131                                  self.POST, self.public_url,
2132                                  t="start-deep-check", ophandle="123")
2133         d.addCallback(self.wait_for_operation, "123")
2134         def _check_json(data):
2135             self.failUnlessReallyEqual(data["finished"], True)
2136             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2137             self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2138         d.addCallback(_check_json)
2139         d.addCallback(self.get_operation_results, "123", "html")
2140         def _check_html(res):
2141             self.failUnless("Objects Checked: <span>8</span>" in res)
2142             self.failUnless("Objects Healthy: <span>8</span>" in res)
2143         d.addCallback(_check_html)
2144
2145         d.addCallback(lambda res:
2146                       self.GET("/operations/123/"))
2147         d.addCallback(_check_html) # should be the same as without the slash
2148
2149         d.addCallback(lambda res:
2150                       self.shouldFail2(error.Error, "one", "404 Not Found",
2151                                        "No detailed results for SI bogus",
2152                                        self.GET, "/operations/123/bogus"))
2153
2154         foo_si = self._foo_node.get_storage_index()
2155         foo_si_s = base32.b2a(foo_si)
2156         d.addCallback(lambda res:
2157                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2158         def _check_foo_json(res):
2159             data = simplejson.loads(res)
2160             self.failUnlessEqual(data["storage-index"], foo_si_s)
2161             self.failUnless(data["results"]["healthy"])
2162         d.addCallback(_check_foo_json)
2163         return d
2164
2165     def test_POST_DIRURL_deepcheck_and_repair(self):
2166         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2167                       ophandle="124", output="json", followRedirect=True)
2168         d.addCallback(self.wait_for_operation, "124")
2169         def _check_json(data):
2170             self.failUnlessReallyEqual(data["finished"], True)
2171             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2172             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2173             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2174             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2175             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2176             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2177             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2178             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2179             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2180             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2181         d.addCallback(_check_json)
2182         d.addCallback(self.get_operation_results, "124", "html")
2183         def _check_html(res):
2184             self.failUnless("Objects Checked: <span>8</span>" in res)
2185
2186             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2187             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2188             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2189
2190             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2191             self.failUnless("Repairs Successful: <span>0</span>" in res)
2192             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2193
2194             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2195             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2196             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2197         d.addCallback(_check_html)
2198         return d
2199
2200     def test_POST_FILEURL_bad_t(self):
2201         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2202                              "POST to file: bad t=bogus",
2203                              self.POST, self.public_url + "/foo/bar.txt",
2204                              t="bogus")
2205         return d
2206
2207     def test_POST_mkdir(self): # return value?
2208         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2209         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2210         d.addCallback(self.failUnlessNodeKeysAre, [])
2211         return d
2212
2213     def test_POST_mkdir_initial_children(self):
2214         (newkids, caps) = self._create_initial_children()
2215         d = self.POST2(self.public_url +
2216                        "/foo?t=mkdir-with-children&name=newdir",
2217                        simplejson.dumps(newkids))
2218         d.addCallback(lambda res:
2219                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2220         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2221         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2222         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2223         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2224         return d
2225
2226     def test_POST_mkdir_immutable(self):
2227         (newkids, caps) = self._create_immutable_children()
2228         d = self.POST2(self.public_url +
2229                        "/foo?t=mkdir-immutable&name=newdir",
2230                        simplejson.dumps(newkids))
2231         d.addCallback(lambda res:
2232                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2233         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2234         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2235         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2236         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2237         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2238         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2239         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2240         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2241         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2242         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2243         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2244         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2245         return d
2246
2247     def test_POST_mkdir_immutable_bad(self):
2248         (newkids, caps) = self._create_initial_children()
2249         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2250                              "400 Bad Request",
2251                              "needed to be immutable but was not",
2252                              self.POST2,
2253                              self.public_url +
2254                              "/foo?t=mkdir-immutable&name=newdir",
2255                              simplejson.dumps(newkids))
2256         return d
2257
2258     def test_POST_mkdir_2(self):
2259         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2260         d.addCallback(lambda res:
2261                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2262         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2263         d.addCallback(self.failUnlessNodeKeysAre, [])
2264         return d
2265
2266     def test_POST_mkdirs_2(self):
2267         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2268         d.addCallback(lambda res:
2269                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2270         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2271         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2272         d.addCallback(self.failUnlessNodeKeysAre, [])
2273         return d
2274
2275     def test_POST_mkdir_no_parentdir_noredirect(self):
2276         d = self.POST("/uri?t=mkdir")
2277         def _after_mkdir(res):
2278             uri.DirectoryURI.init_from_string(res)
2279         d.addCallback(_after_mkdir)
2280         return d
2281
2282     def test_POST_mkdir_no_parentdir_noredirect2(self):
2283         # make sure form-based arguments (as on the welcome page) still work
2284         d = self.POST("/uri", t="mkdir")
2285         def _after_mkdir(res):
2286             uri.DirectoryURI.init_from_string(res)
2287         d.addCallback(_after_mkdir)
2288         d.addErrback(self.explain_web_error)
2289         return d
2290
2291     def test_POST_mkdir_no_parentdir_redirect(self):
2292         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2293         d.addBoth(self.shouldRedirect, None, statuscode='303')
2294         def _check_target(target):
2295             target = urllib.unquote(target)
2296             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2297         d.addCallback(_check_target)
2298         return d
2299
2300     def test_POST_mkdir_no_parentdir_redirect2(self):
2301         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2302         d.addBoth(self.shouldRedirect, None, statuscode='303')
2303         def _check_target(target):
2304             target = urllib.unquote(target)
2305             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2306         d.addCallback(_check_target)
2307         d.addErrback(self.explain_web_error)
2308         return d
2309
2310     def _make_readonly(self, u):
2311         ro_uri = uri.from_string(u).get_readonly()
2312         if ro_uri is None:
2313             return None
2314         return ro_uri.to_string()
2315
2316     def _create_initial_children(self):
2317         contents, n, filecap1 = self.makefile(12)
2318         md1 = {"metakey1": "metavalue1"}
2319         filecap2 = make_mutable_file_uri()
2320         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2321         filecap3 = node3.get_readonly_uri()
2322         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2323         dircap = DirectoryNode(node4, None, None).get_uri()
2324         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2325         emptydircap = "URI:DIR2-LIT:"
2326         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2327                                                       "ro_uri": self._make_readonly(filecap1),
2328                                                       "metadata": md1, }],
2329                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2330                                                       "ro_uri": self._make_readonly(filecap2)}],
2331                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2332                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2333                                                       "ro_uri": unknown_rocap}],
2334                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2335                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2336                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2337                                                       "ro_uri": self._make_readonly(dircap)}],
2338                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2339                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2340                    }
2341         return newkids, {'filecap1': filecap1,
2342                          'filecap2': filecap2,
2343                          'filecap3': filecap3,
2344                          'unknown_rwcap': unknown_rwcap,
2345                          'unknown_rocap': unknown_rocap,
2346                          'unknown_immcap': unknown_immcap,
2347                          'dircap': dircap,
2348                          'litdircap': litdircap,
2349                          'emptydircap': emptydircap}
2350
2351     def _create_immutable_children(self):
2352         contents, n, filecap1 = self.makefile(12)
2353         md1 = {"metakey1": "metavalue1"}
2354         tnode = create_chk_filenode("immutable directory contents\n"*10)
2355         dnode = DirectoryNode(tnode, None, None)
2356         assert not dnode.is_mutable()
2357         immdircap = dnode.get_uri()
2358         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2359         emptydircap = "URI:DIR2-LIT:"
2360         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2361                                                       "metadata": md1, }],
2362                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2363                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2364                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2365                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2366                    }
2367         return newkids, {'filecap1': filecap1,
2368                          'unknown_immcap': unknown_immcap,
2369                          'immdircap': immdircap,
2370                          'litdircap': litdircap,
2371                          'emptydircap': emptydircap}
2372
2373     def test_POST_mkdir_no_parentdir_initial_children(self):
2374         (newkids, caps) = self._create_initial_children()
2375         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2376         def _after_mkdir(res):
2377             self.failUnless(res.startswith("URI:DIR"), res)
2378             n = self.s.create_node_from_uri(res)
2379             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2380             d2.addCallback(lambda ign:
2381                            self.failUnlessROChildURIIs(n, u"child-imm",
2382                                                        caps['filecap1']))
2383             d2.addCallback(lambda ign:
2384                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2385                                                        caps['filecap2']))
2386             d2.addCallback(lambda ign:
2387                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2388                                                        caps['filecap3']))
2389             d2.addCallback(lambda ign:
2390                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2391                                                        caps['unknown_rwcap']))
2392             d2.addCallback(lambda ign:
2393                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2394                                                        caps['unknown_rocap']))
2395             d2.addCallback(lambda ign:
2396                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2397                                                        caps['unknown_immcap']))
2398             d2.addCallback(lambda ign:
2399                            self.failUnlessRWChildURIIs(n, u"dirchild",
2400                                                        caps['dircap']))
2401             return d2
2402         d.addCallback(_after_mkdir)
2403         return d
2404
2405     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2406         # the regular /uri?t=mkdir operation is specified to ignore its body.
2407         # Only t=mkdir-with-children pays attention to it.
2408         (newkids, caps) = self._create_initial_children()
2409         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2410                                  400, "Bad Request",
2411                                  "t=mkdir does not accept children=, "
2412                                  "try t=mkdir-with-children instead",
2413                                  self.POST2, "/uri?t=mkdir", # without children
2414                                  simplejson.dumps(newkids))
2415         return d
2416
2417     def test_POST_noparent_bad(self):
2418         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2419                                  "/uri accepts only PUT, PUT?t=mkdir, "
2420                                  "POST?t=upload, and POST?t=mkdir",
2421                                  self.POST, "/uri?t=bogus")
2422         return d
2423
2424     def test_POST_mkdir_no_parentdir_immutable(self):
2425         (newkids, caps) = self._create_immutable_children()
2426         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2427         def _after_mkdir(res):
2428             self.failUnless(res.startswith("URI:DIR"), res)
2429             n = self.s.create_node_from_uri(res)
2430             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2431             d2.addCallback(lambda ign:
2432                            self.failUnlessROChildURIIs(n, u"child-imm",
2433                                                           caps['filecap1']))
2434             d2.addCallback(lambda ign:
2435                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2436                                                           caps['unknown_immcap']))
2437             d2.addCallback(lambda ign:
2438                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2439                                                           caps['immdircap']))
2440             d2.addCallback(lambda ign:
2441                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2442                                                           caps['litdircap']))
2443             d2.addCallback(lambda ign:
2444                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2445                                                           caps['emptydircap']))
2446             return d2
2447         d.addCallback(_after_mkdir)
2448         return d
2449
2450     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2451         (newkids, caps) = self._create_initial_children()
2452         d = self.shouldFail2(error.Error,
2453                              "test_POST_mkdir_no_parentdir_immutable_bad",
2454                              "400 Bad Request",
2455                              "needed to be immutable but was not",
2456                              self.POST2,
2457                              "/uri?t=mkdir-immutable",
2458                              simplejson.dumps(newkids))
2459         return d
2460
2461     def test_welcome_page_mkdir_button(self):
2462         # Fetch the welcome page.
2463         d = self.GET("/")
2464         def _after_get_welcome_page(res):
2465             MKDIR_BUTTON_RE = re.compile(
2466                 '<form action="([^"]*)" method="post".*?'
2467                 '<input type="hidden" name="t" value="([^"]*)" />'
2468                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2469                 '<input type="submit" value="Create a directory" />',
2470                 re.I)
2471             mo = MKDIR_BUTTON_RE.search(res)
2472             formaction = mo.group(1)
2473             formt = mo.group(2)
2474             formaname = mo.group(3)
2475             formavalue = mo.group(4)
2476             return (formaction, formt, formaname, formavalue)
2477         d.addCallback(_after_get_welcome_page)
2478         def _after_parse_form(res):
2479             (formaction, formt, formaname, formavalue) = res
2480             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2481         d.addCallback(_after_parse_form)
2482         d.addBoth(self.shouldRedirect, None, statuscode='303')
2483         return d
2484
2485     def test_POST_mkdir_replace(self): # return value?
2486         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2487         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2488         d.addCallback(self.failUnlessNodeKeysAre, [])
2489         return d
2490
2491     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2492         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2493         d.addBoth(self.shouldFail, error.Error,
2494                   "POST_mkdir_no_replace_queryarg",
2495                   "409 Conflict",
2496                   "There was already a child by that name, and you asked me "
2497                   "to not replace it")
2498         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2499         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2500         return d
2501
2502     def test_POST_mkdir_no_replace_field(self): # return value?
2503         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2504                       replace="false")
2505         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2506                   "409 Conflict",
2507                   "There was already a child by that name, and you asked me "
2508                   "to not replace it")
2509         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2510         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2511         return d
2512
2513     def test_POST_mkdir_whendone_field(self):
2514         d = self.POST(self.public_url + "/foo",
2515                       t="mkdir", name="newdir", when_done="/THERE")
2516         d.addBoth(self.shouldRedirect, "/THERE")
2517         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2518         d.addCallback(self.failUnlessNodeKeysAre, [])
2519         return d
2520
2521     def test_POST_mkdir_whendone_queryarg(self):
2522         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2523                       t="mkdir", name="newdir")
2524         d.addBoth(self.shouldRedirect, "/THERE")
2525         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2526         d.addCallback(self.failUnlessNodeKeysAre, [])
2527         return d
2528
2529     def test_POST_bad_t(self):
2530         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2531                              "POST to a directory with bad t=BOGUS",
2532                              self.POST, self.public_url + "/foo", t="BOGUS")
2533         return d
2534
2535     def test_POST_set_children(self, command_name="set_children"):
2536         contents9, n9, newuri9 = self.makefile(9)
2537         contents10, n10, newuri10 = self.makefile(10)
2538         contents11, n11, newuri11 = self.makefile(11)
2539
2540         reqbody = """{
2541                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2542                                                 "size": 0,
2543                                                 "metadata": {
2544                                                   "ctime": 1002777696.7564139,
2545                                                   "mtime": 1002777696.7564139
2546                                                  }
2547                                                } ],
2548                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2549                                                 "size": 1,
2550                                                 "metadata": {
2551                                                   "ctime": 1002777696.7564139,
2552                                                   "mtime": 1002777696.7564139
2553                                                  }
2554                                                } ],
2555                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2556                                                 "size": 2,
2557                                                 "metadata": {
2558                                                   "ctime": 1002777696.7564139,
2559                                                   "mtime": 1002777696.7564139
2560                                                  }
2561                                                } ]
2562                     }""" % (newuri9, newuri10, newuri11)
2563
2564         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2565
2566         d = client.getPage(url, method="POST", postdata=reqbody)
2567         def _then(res):
2568             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2569             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2570             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2571
2572         d.addCallback(_then)
2573         d.addErrback(self.dump_error)
2574         return d
2575
2576     def test_POST_set_children_with_hyphen(self):
2577         return self.test_POST_set_children(command_name="set-children")
2578
2579     def test_POST_link_uri(self):
2580         contents, n, newuri = self.makefile(8)
2581         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2582         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2583         d.addCallback(lambda res:
2584                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2585                                                       contents))
2586         return d
2587
2588     def test_POST_link_uri_replace(self):
2589         contents, n, newuri = self.makefile(8)
2590         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2591         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2592         d.addCallback(lambda res:
2593                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2594                                                       contents))
2595         return d
2596
2597     def test_POST_link_uri_unknown_bad(self):
2598         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2599         d.addBoth(self.shouldFail, error.Error,
2600                   "POST_link_uri_unknown_bad",
2601                   "400 Bad Request",
2602                   "unknown cap in a write slot")
2603         return d
2604
2605     def test_POST_link_uri_unknown_ro_good(self):
2606         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2607         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2608         return d
2609
2610     def test_POST_link_uri_unknown_imm_good(self):
2611         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2612         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2613         return d
2614
2615     def test_POST_link_uri_no_replace_queryarg(self):
2616         contents, n, newuri = self.makefile(8)
2617         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2618                       name="bar.txt", uri=newuri)
2619         d.addBoth(self.shouldFail, error.Error,
2620                   "POST_link_uri_no_replace_queryarg",
2621                   "409 Conflict",
2622                   "There was already a child by that name, and you asked me "
2623                   "to not replace it")
2624         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2625         d.addCallback(self.failUnlessIsBarDotTxt)
2626         return d
2627
2628     def test_POST_link_uri_no_replace_field(self):
2629         contents, n, newuri = self.makefile(8)
2630         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2631                       name="bar.txt", uri=newuri)
2632         d.addBoth(self.shouldFail, error.Error,
2633                   "POST_link_uri_no_replace_field",
2634                   "409 Conflict",
2635                   "There was already a child by that name, and you asked me "
2636                   "to not replace it")
2637         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2638         d.addCallback(self.failUnlessIsBarDotTxt)
2639         return d
2640
2641     def test_POST_delete(self):
2642         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2643         d.addCallback(lambda res: self._foo_node.list())
2644         def _check(children):
2645             self.failIf(u"bar.txt" in children)
2646         d.addCallback(_check)
2647         return d
2648
2649     def test_POST_rename_file(self):
2650         d = self.POST(self.public_url + "/foo", t="rename",
2651                       from_name="bar.txt", to_name='wibble.txt')
2652         d.addCallback(lambda res:
2653                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2654         d.addCallback(lambda res:
2655                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2656         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2657         d.addCallback(self.failUnlessIsBarDotTxt)
2658         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2659         d.addCallback(self.failUnlessIsBarJSON)
2660         return d
2661
2662     def test_POST_rename_file_redundant(self):
2663         d = self.POST(self.public_url + "/foo", t="rename",
2664                       from_name="bar.txt", to_name='bar.txt')
2665         d.addCallback(lambda res:
2666                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2667         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2668         d.addCallback(self.failUnlessIsBarDotTxt)
2669         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2670         d.addCallback(self.failUnlessIsBarJSON)
2671         return d
2672
2673     def test_POST_rename_file_replace(self):
2674         # rename a file and replace a directory with it
2675         d = self.POST(self.public_url + "/foo", t="rename",
2676                       from_name="bar.txt", to_name='empty')
2677         d.addCallback(lambda res:
2678                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2679         d.addCallback(lambda res:
2680                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2681         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2682         d.addCallback(self.failUnlessIsBarDotTxt)
2683         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2684         d.addCallback(self.failUnlessIsBarJSON)
2685         return d
2686
2687     def test_POST_rename_file_no_replace_queryarg(self):
2688         # rename a file and replace a directory with it
2689         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2690                       from_name="bar.txt", to_name='empty')
2691         d.addBoth(self.shouldFail, error.Error,
2692                   "POST_rename_file_no_replace_queryarg",
2693                   "409 Conflict",
2694                   "There was already a child by that name, and you asked me "
2695                   "to not replace it")
2696         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2697         d.addCallback(self.failUnlessIsEmptyJSON)
2698         return d
2699
2700     def test_POST_rename_file_no_replace_field(self):
2701         # rename a file and replace a directory with it
2702         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2703                       from_name="bar.txt", to_name='empty')
2704         d.addBoth(self.shouldFail, error.Error,
2705                   "POST_rename_file_no_replace_field",
2706                   "409 Conflict",
2707                   "There was already a child by that name, and you asked me "
2708                   "to not replace it")
2709         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2710         d.addCallback(self.failUnlessIsEmptyJSON)
2711         return d
2712
2713     def failUnlessIsEmptyJSON(self, res):
2714         data = simplejson.loads(res)
2715         self.failUnlessEqual(data[0], "dirnode", data)
2716         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2717
2718     def test_POST_rename_file_slash_fail(self):
2719         d = self.POST(self.public_url + "/foo", t="rename",
2720                       from_name="bar.txt", to_name='kirk/spock.txt')
2721         d.addBoth(self.shouldFail, error.Error,
2722                   "test_POST_rename_file_slash_fail",
2723                   "400 Bad Request",
2724                   "to_name= may not contain a slash",
2725                   )
2726         d.addCallback(lambda res:
2727                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2728         return d
2729
2730     def test_POST_rename_dir(self):
2731         d = self.POST(self.public_url, t="rename",
2732                       from_name="foo", to_name='plunk')
2733         d.addCallback(lambda res:
2734                       self.failIfNodeHasChild(self.public_root, u"foo"))
2735         d.addCallback(lambda res:
2736                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2737         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2738         d.addCallback(self.failUnlessIsFooJSON)
2739         return d
2740
2741     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2742         """ If target is not None then the redirection has to go to target.  If
2743         statuscode is not None then the redirection has to be accomplished with
2744         that HTTP status code."""
2745         if not isinstance(res, failure.Failure):
2746             to_where = (target is None) and "somewhere" or ("to " + target)
2747             self.fail("%s: we were expecting to get redirected %s, not get an"
2748                       " actual page: %s" % (which, to_where, res))
2749         res.trap(error.PageRedirect)
2750         if statuscode is not None:
2751             self.failUnlessReallyEqual(res.value.status, statuscode,
2752                                        "%s: not a redirect" % which)
2753         if target is not None:
2754             # the PageRedirect does not seem to capture the uri= query arg
2755             # properly, so we can't check for it.
2756             realtarget = self.webish_url + target
2757             self.failUnlessReallyEqual(res.value.location, realtarget,
2758                                        "%s: wrong target" % which)
2759         return res.value.location
2760
2761     def test_GET_URI_form(self):
2762         base = "/uri?uri=%s" % self._bar_txt_uri
2763         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2764         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2765         d = self.GET(base)
2766         d.addBoth(self.shouldRedirect, targetbase)
2767         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2768         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2769         d.addCallback(lambda res: self.GET(base+"&t=json"))
2770         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2771         d.addCallback(self.log, "about to get file by uri")
2772         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2773         d.addCallback(self.failUnlessIsBarDotTxt)
2774         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2775         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2776                                            followRedirect=True))
2777         d.addCallback(self.failUnlessIsFooJSON)
2778         d.addCallback(self.log, "got dir by uri")
2779
2780         return d
2781
2782     def test_GET_URI_form_bad(self):
2783         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2784                              "400 Bad Request", "GET /uri requires uri=",
2785                              self.GET, "/uri")
2786         return d
2787
2788     def test_GET_rename_form(self):
2789         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2790                      followRedirect=True)
2791         def _check(res):
2792             self.failUnless('name="when_done" value="."' in res, res)
2793             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2794         d.addCallback(_check)
2795         return d
2796
2797     def log(self, res, msg):
2798         #print "MSG: %s  RES: %s" % (msg, res)
2799         log.msg(msg)
2800         return res
2801
2802     def test_GET_URI_URL(self):
2803         base = "/uri/%s" % self._bar_txt_uri
2804         d = self.GET(base)
2805         d.addCallback(self.failUnlessIsBarDotTxt)
2806         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2807         d.addCallback(self.failUnlessIsBarDotTxt)
2808         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2809         d.addCallback(self.failUnlessIsBarDotTxt)
2810         return d
2811
2812     def test_GET_URI_URL_dir(self):
2813         base = "/uri/%s?t=json" % self._foo_uri
2814         d = self.GET(base)
2815         d.addCallback(self.failUnlessIsFooJSON)
2816         return d
2817
2818     def test_GET_URI_URL_missing(self):
2819         base = "/uri/%s" % self._bad_file_uri
2820         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2821                                  http.GONE, None, "NotEnoughSharesError",
2822                                  self.GET, base)
2823         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2824         # here? we must arrange for a download to fail after target.open()
2825         # has been called, and then inspect the response to see that it is
2826         # shorter than we expected.
2827         return d
2828
2829     def test_PUT_DIRURL_uri(self):
2830         d = self.s.create_dirnode()
2831         def _made_dir(dn):
2832             new_uri = dn.get_uri()
2833             # replace /foo with a new (empty) directory
2834             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2835             d.addCallback(lambda res:
2836                           self.failUnlessReallyEqual(res.strip(), new_uri))
2837             d.addCallback(lambda res:
2838                           self.failUnlessRWChildURIIs(self.public_root,
2839                                                       u"foo",
2840                                                       new_uri))
2841             return d
2842         d.addCallback(_made_dir)
2843         return d
2844
2845     def test_PUT_DIRURL_uri_noreplace(self):
2846         d = self.s.create_dirnode()
2847         def _made_dir(dn):
2848             new_uri = dn.get_uri()
2849             # replace /foo with a new (empty) directory, but ask that
2850             # replace=false, so it should fail
2851             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2852                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2853                                  self.PUT,
2854                                  self.public_url + "/foo?t=uri&replace=false",
2855                                  new_uri)
2856             d.addCallback(lambda res:
2857                           self.failUnlessRWChildURIIs(self.public_root,
2858                                                       u"foo",
2859                                                       self._foo_uri))
2860             return d
2861         d.addCallback(_made_dir)
2862         return d
2863
2864     def test_PUT_DIRURL_bad_t(self):
2865         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2866                                  "400 Bad Request", "PUT to a directory",
2867                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2868         d.addCallback(lambda res:
2869                       self.failUnlessRWChildURIIs(self.public_root,
2870                                                   u"foo",
2871                                                   self._foo_uri))
2872         return d
2873
2874     def test_PUT_NEWFILEURL_uri(self):
2875         contents, n, new_uri = self.makefile(8)
2876         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2877         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2878         d.addCallback(lambda res:
2879                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2880                                                       contents))
2881         return d
2882
2883     def test_PUT_NEWFILEURL_uri_replace(self):
2884         contents, n, new_uri = self.makefile(8)
2885         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2886         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2887         d.addCallback(lambda res:
2888                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2889                                                       contents))
2890         return d
2891
2892     def test_PUT_NEWFILEURL_uri_no_replace(self):
2893         contents, n, new_uri = self.makefile(8)
2894         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2895         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2896                   "409 Conflict",
2897                   "There was already a child by that name, and you asked me "
2898                   "to not replace it")
2899         return d
2900
2901     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2902         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2903         d.addBoth(self.shouldFail, error.Error,
2904                   "POST_put_uri_unknown_bad",
2905                   "400 Bad Request",
2906                   "unknown cap in a write slot")
2907         return d
2908
2909     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2910         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2911         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2912                       u"put-future-ro.txt")
2913         return d
2914
2915     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2916         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2917         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2918                       u"put-future-imm.txt")
2919         return d
2920
2921     def test_PUT_NEWFILE_URI(self):
2922         file_contents = "New file contents here\n"
2923         d = self.PUT("/uri", file_contents)
2924         def _check(uri):
2925             assert isinstance(uri, str), uri
2926             self.failUnless(uri in FakeCHKFileNode.all_contents)
2927             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2928                                        file_contents)
2929             return self.GET("/uri/%s" % uri)
2930         d.addCallback(_check)
2931         def _check2(res):
2932             self.failUnlessReallyEqual(res, file_contents)
2933         d.addCallback(_check2)
2934         return d
2935
2936     def test_PUT_NEWFILE_URI_not_mutable(self):
2937         file_contents = "New file contents here\n"
2938         d = self.PUT("/uri?mutable=false", file_contents)
2939         def _check(uri):
2940             assert isinstance(uri, str), uri
2941             self.failUnless(uri in FakeCHKFileNode.all_contents)
2942             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2943                                        file_contents)
2944             return self.GET("/uri/%s" % uri)
2945         d.addCallback(_check)
2946         def _check2(res):
2947             self.failUnlessReallyEqual(res, file_contents)
2948         d.addCallback(_check2)
2949         return d
2950
2951     def test_PUT_NEWFILE_URI_only_PUT(self):
2952         d = self.PUT("/uri?t=bogus", "")
2953         d.addBoth(self.shouldFail, error.Error,
2954                   "PUT_NEWFILE_URI_only_PUT",
2955                   "400 Bad Request",
2956                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2957         return d
2958
2959     def test_PUT_NEWFILE_URI_mutable(self):
2960         file_contents = "New file contents here\n"
2961         d = self.PUT("/uri?mutable=true", file_contents)
2962         def _check1(filecap):
2963             filecap = filecap.strip()
2964             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2965             self.filecap = filecap
2966             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2967             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2968             n = self.s.create_node_from_uri(filecap)
2969             return n.download_best_version()
2970         d.addCallback(_check1)
2971         def _check2(data):
2972             self.failUnlessReallyEqual(data, file_contents)
2973             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2974         d.addCallback(_check2)
2975         def _check3(res):
2976             self.failUnlessReallyEqual(res, file_contents)
2977         d.addCallback(_check3)
2978         return d
2979
2980     def test_PUT_mkdir(self):
2981         d = self.PUT("/uri?t=mkdir", "")
2982         def _check(uri):
2983             n = self.s.create_node_from_uri(uri.strip())
2984             d2 = self.failUnlessNodeKeysAre(n, [])
2985             d2.addCallback(lambda res:
2986                            self.GET("/uri/%s?t=json" % uri))
2987             return d2
2988         d.addCallback(_check)
2989         d.addCallback(self.failUnlessIsEmptyJSON)
2990         return d
2991
2992     def test_POST_check(self):
2993         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2994         def _done(res):
2995             # this returns a string form of the results, which are probably
2996             # None since we're using fake filenodes.
2997             # TODO: verify that the check actually happened, by changing
2998             # FakeCHKFileNode to count how many times .check() has been
2999             # called.
3000             pass
3001         d.addCallback(_done)
3002         return d
3003
3004     def test_bad_method(self):
3005         url = self.webish_url + self.public_url + "/foo/bar.txt"
3006         d = self.shouldHTTPError("test_bad_method",
3007                                  501, "Not Implemented",
3008                                  "I don't know how to treat a BOGUS request.",
3009                                  client.getPage, url, method="BOGUS")
3010         return d
3011
3012     def test_short_url(self):
3013         url = self.webish_url + "/uri"
3014         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3015                                  "I don't know how to treat a DELETE request.",
3016                                  client.getPage, url, method="DELETE")
3017         return d
3018
3019     def test_ophandle_bad(self):
3020         url = self.webish_url + "/operations/bogus?t=status"
3021         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3022                                  "unknown/expired handle 'bogus'",
3023                                  client.getPage, url)
3024         return d
3025
3026     def test_ophandle_cancel(self):
3027         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3028                       followRedirect=True)
3029         d.addCallback(lambda ignored:
3030                       self.GET("/operations/128?t=status&output=JSON"))
3031         def _check1(res):
3032             data = simplejson.loads(res)
3033             self.failUnless("finished" in data, res)
3034             monitor = self.ws.root.child_operations.handles["128"][0]
3035             d = self.POST("/operations/128?t=cancel&output=JSON")
3036             def _check2(res):
3037                 data = simplejson.loads(res)
3038                 self.failUnless("finished" in data, res)
3039                 # t=cancel causes the handle to be forgotten
3040                 self.failUnless(monitor.is_cancelled())
3041             d.addCallback(_check2)
3042             return d
3043         d.addCallback(_check1)
3044         d.addCallback(lambda ignored:
3045                       self.shouldHTTPError("test_ophandle_cancel",
3046                                            404, "404 Not Found",
3047                                            "unknown/expired handle '128'",
3048                                            self.GET,
3049                                            "/operations/128?t=status&output=JSON"))
3050         return d
3051
3052     def test_ophandle_retainfor(self):
3053         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3054                       followRedirect=True)
3055         d.addCallback(lambda ignored:
3056                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3057         def _check1(res):
3058             data = simplejson.loads(res)
3059             self.failUnless("finished" in data, res)
3060         d.addCallback(_check1)
3061         # the retain-for=0 will cause the handle to be expired very soon
3062         d.addCallback(lambda ign:
3063             self.clock.advance(2.0))
3064         d.addCallback(lambda ignored:
3065                       self.shouldHTTPError("test_ophandle_retainfor",
3066                                            404, "404 Not Found",
3067                                            "unknown/expired handle '129'",
3068                                            self.GET,
3069                                            "/operations/129?t=status&output=JSON"))
3070         return d
3071
3072     def test_ophandle_release_after_complete(self):
3073         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3074                       followRedirect=True)
3075         d.addCallback(self.wait_for_operation, "130")
3076         d.addCallback(lambda ignored:
3077                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3078         # the release-after-complete=true will cause the handle to be expired
3079         d.addCallback(lambda ignored:
3080                       self.shouldHTTPError("test_ophandle_release_after_complete",
3081                                            404, "404 Not Found",
3082                                            "unknown/expired handle '130'",
3083                                            self.GET,
3084                                            "/operations/130?t=status&output=JSON"))
3085         return d
3086
3087     def test_uncollected_ophandle_expiration(self):
3088         # uncollected ophandles should expire after 4 days
3089         def _make_uncollected_ophandle(ophandle):
3090             d = self.POST(self.public_url +
3091                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3092                           followRedirect=False)
3093             # When we start the operation, the webapi server will want
3094             # to redirect us to the page for the ophandle, so we get
3095             # confirmation that the operation has started. If the
3096             # manifest operation has finished by the time we get there,
3097             # following that redirect (by setting followRedirect=True
3098             # above) has the side effect of collecting the ophandle that
3099             # we've just created, which means that we can't use the
3100             # ophandle to test the uncollected timeout anymore. So,
3101             # instead, catch the 302 here and don't follow it.
3102             d.addBoth(self.should302, "uncollected_ophandle_creation")
3103             return d
3104         # Create an ophandle, don't collect it, then advance the clock by
3105         # 4 days - 1 second and make sure that the ophandle is still there.
3106         d = _make_uncollected_ophandle(131)
3107         d.addCallback(lambda ign:
3108             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3109         d.addCallback(lambda ign:
3110             self.GET("/operations/131?t=status&output=JSON"))
3111         def _check1(res):
3112             data = simplejson.loads(res)
3113             self.failUnless("finished" in data, res)
3114         d.addCallback(_check1)
3115         # Create an ophandle, don't collect it, then try to collect it
3116         # after 4 days. It should be gone.
3117         d.addCallback(lambda ign:
3118             _make_uncollected_ophandle(132))
3119         d.addCallback(lambda ign:
3120             self.clock.advance(96*60*60))
3121         d.addCallback(lambda ign:
3122             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3123                                  404, "404 Not Found",
3124                                  "unknown/expired handle '132'",
3125                                  self.GET,
3126                                  "/operations/132?t=status&output=JSON"))
3127         return d
3128
3129     def test_collected_ophandle_expiration(self):
3130         # collected ophandles should expire after 1 day
3131         def _make_collected_ophandle(ophandle):
3132             d = self.POST(self.public_url +
3133                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3134                           followRedirect=True)
3135             # By following the initial redirect, we collect the ophandle
3136             # we've just created.
3137             return d
3138         # Create a collected ophandle, then collect it after 23 hours
3139         # and 59 seconds to make sure that it is still there.
3140         d = _make_collected_ophandle(133)
3141         d.addCallback(lambda ign:
3142             self.clock.advance((24*60*60) - 1))
3143         d.addCallback(lambda ign:
3144             self.GET("/operations/133?t=status&output=JSON"))
3145         def _check1(res):
3146             data = simplejson.loads(res)
3147             self.failUnless("finished" in data, res)
3148         d.addCallback(_check1)
3149         # Create another uncollected ophandle, then try to collect it
3150         # after 24 hours to make sure that it is gone.
3151         d.addCallback(lambda ign:
3152             _make_collected_ophandle(134))
3153         d.addCallback(lambda ign:
3154             self.clock.advance(24*60*60))
3155         d.addCallback(lambda ign:
3156             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3157                                  404, "404 Not Found",
3158                                  "unknown/expired handle '134'",
3159                                  self.GET,
3160                                  "/operations/134?t=status&output=JSON"))
3161         return d
3162
3163     def test_incident(self):
3164         d = self.POST("/report_incident", details="eek")
3165         def _done(res):
3166             self.failUnless("Thank you for your report!" in res, res)
3167         d.addCallback(_done)
3168         return d
3169
3170     def test_static(self):
3171         webdir = os.path.join(self.staticdir, "subdir")
3172         fileutil.make_dirs(webdir)
3173         f = open(os.path.join(webdir, "hello.txt"), "wb")
3174         f.write("hello")
3175         f.close()
3176
3177         d = self.GET("/static/subdir/hello.txt")
3178         def _check(res):
3179             self.failUnlessReallyEqual(res, "hello")
3180         d.addCallback(_check)
3181         return d
3182
3183
3184 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3185     def test_load_file(self):
3186         # This will raise an exception unless a well-formed XML file is found under that name.
3187         common.getxmlfile('directory.xhtml').load()
3188
3189     def test_parse_replace_arg(self):
3190         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3191         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3192         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3193                                    "only-files")
3194         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3195                         common.parse_replace_arg, "only_fles")
3196
3197     def test_abbreviate_time(self):
3198         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3199         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3200         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3201         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3202         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3203         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3204
3205     def test_compute_rate(self):
3206         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3207         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3208         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3209         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3210         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3211         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3212         self.shouldFail(AssertionError, "test_compute_rate", "",
3213                         common.compute_rate, -100, 10)
3214         self.shouldFail(AssertionError, "test_compute_rate", "",
3215                         common.compute_rate, 100, -10)
3216
3217         # Sanity check
3218         rate = common.compute_rate(10*1000*1000, 1)
3219         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3220
3221     def test_abbreviate_rate(self):
3222         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3223         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3224         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3225         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3226
3227     def test_abbreviate_size(self):
3228         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3229         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3230         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3231         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3232         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3233
3234     def test_plural(self):
3235         def convert(s):
3236             return "%d second%s" % (s, status.plural(s))
3237         self.failUnlessReallyEqual(convert(0), "0 seconds")
3238         self.failUnlessReallyEqual(convert(1), "1 second")
3239         self.failUnlessReallyEqual(convert(2), "2 seconds")
3240         def convert2(s):
3241             return "has share%s: %s" % (status.plural(s), ",".join(s))
3242         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3243         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3244         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3245
3246
3247 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3248
3249     def CHECK(self, ign, which, args, clientnum=0):
3250         fileurl = self.fileurls[which]
3251         url = fileurl + "?" + args
3252         return self.GET(url, method="POST", clientnum=clientnum)
3253
3254     def test_filecheck(self):
3255         self.basedir = "web/Grid/filecheck"
3256         self.set_up_grid()
3257         c0 = self.g.clients[0]
3258         self.uris = {}
3259         DATA = "data" * 100
3260         d = c0.upload(upload.Data(DATA, convergence=""))
3261         def _stash_uri(ur, which):
3262             self.uris[which] = ur.uri
3263         d.addCallback(_stash_uri, "good")
3264         d.addCallback(lambda ign:
3265                       c0.upload(upload.Data(DATA+"1", convergence="")))
3266         d.addCallback(_stash_uri, "sick")
3267         d.addCallback(lambda ign:
3268                       c0.upload(upload.Data(DATA+"2", convergence="")))
3269         d.addCallback(_stash_uri, "dead")
3270         def _stash_mutable_uri(n, which):
3271             self.uris[which] = n.get_uri()
3272             assert isinstance(self.uris[which], str)
3273         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3274         d.addCallback(_stash_mutable_uri, "corrupt")
3275         d.addCallback(lambda ign:
3276                       c0.upload(upload.Data("literal", convergence="")))
3277         d.addCallback(_stash_uri, "small")
3278         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3279         d.addCallback(_stash_mutable_uri, "smalldir")
3280
3281         def _compute_fileurls(ignored):
3282             self.fileurls = {}
3283             for which in self.uris:
3284                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3285         d.addCallback(_compute_fileurls)
3286
3287         def _clobber_shares(ignored):
3288             good_shares = self.find_uri_shares(self.uris["good"])
3289             self.failUnlessReallyEqual(len(good_shares), 10)
3290             sick_shares = self.find_uri_shares(self.uris["sick"])
3291             os.unlink(sick_shares[0][2])
3292             dead_shares = self.find_uri_shares(self.uris["dead"])
3293             for i in range(1, 10):
3294                 os.unlink(dead_shares[i][2])
3295             c_shares = self.find_uri_shares(self.uris["corrupt"])
3296             cso = CorruptShareOptions()
3297             cso.stdout = StringIO()
3298             cso.parseOptions([c_shares[0][2]])
3299             corrupt_share(cso)
3300         d.addCallback(_clobber_shares)
3301
3302         d.addCallback(self.CHECK, "good", "t=check")
3303         def _got_html_good(res):
3304             self.failUnless("Healthy" in res, res)
3305             self.failIf("Not Healthy" in res, res)
3306         d.addCallback(_got_html_good)
3307         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3308         def _got_html_good_return_to(res):
3309             self.failUnless("Healthy" in res, res)
3310             self.failIf("Not Healthy" in res, res)
3311             self.failUnless('<a href="somewhere">Return to file'
3312                             in res, res)
3313         d.addCallback(_got_html_good_return_to)
3314         d.addCallback(self.CHECK, "good", "t=check&output=json")
3315         def _got_json_good(res):
3316             r = simplejson.loads(res)
3317             self.failUnlessEqual(r["summary"], "Healthy")
3318             self.failUnless(r["results"]["healthy"])
3319             self.failIf(r["results"]["needs-rebalancing"])
3320             self.failUnless(r["results"]["recoverable"])
3321         d.addCallback(_got_json_good)
3322
3323         d.addCallback(self.CHECK, "small", "t=check")
3324         def _got_html_small(res):
3325             self.failUnless("Literal files are always healthy" in res, res)
3326             self.failIf("Not Healthy" in res, res)
3327         d.addCallback(_got_html_small)
3328         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3329         def _got_html_small_return_to(res):
3330             self.failUnless("Literal files are always healthy" in res, res)
3331             self.failIf("Not Healthy" in res, res)
3332             self.failUnless('<a href="somewhere">Return to file'
3333                             in res, res)
3334         d.addCallback(_got_html_small_return_to)
3335         d.addCallback(self.CHECK, "small", "t=check&output=json")
3336         def _got_json_small(res):
3337             r = simplejson.loads(res)
3338             self.failUnlessEqual(r["storage-index"], "")
3339             self.failUnless(r["results"]["healthy"])
3340         d.addCallback(_got_json_small)
3341
3342         d.addCallback(self.CHECK, "smalldir", "t=check")
3343         def _got_html_smalldir(res):
3344             self.failUnless("Literal files are always healthy" in res, res)
3345             self.failIf("Not Healthy" in res, res)
3346         d.addCallback(_got_html_smalldir)
3347         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3348         def _got_json_smalldir(res):
3349             r = simplejson.loads(res)
3350             self.failUnlessEqual(r["storage-index"], "")
3351             self.failUnless(r["results"]["healthy"])
3352         d.addCallback(_got_json_smalldir)
3353
3354         d.addCallback(self.CHECK, "sick", "t=check")
3355         def _got_html_sick(res):
3356             self.failUnless("Not Healthy" in res, res)
3357         d.addCallback(_got_html_sick)
3358         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3359         def _got_json_sick(res):
3360             r = simplejson.loads(res)
3361             self.failUnlessEqual(r["summary"],
3362                                  "Not Healthy: 9 shares (enc 3-of-10)")
3363             self.failIf(r["results"]["healthy"])
3364             self.failIf(r["results"]["needs-rebalancing"])
3365             self.failUnless(r["results"]["recoverable"])
3366         d.addCallback(_got_json_sick)
3367
3368         d.addCallback(self.CHECK, "dead", "t=check")
3369         def _got_html_dead(res):
3370             self.failUnless("Not Healthy" in res, res)
3371         d.addCallback(_got_html_dead)
3372         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3373         def _got_json_dead(res):
3374             r = simplejson.loads(res)
3375             self.failUnlessEqual(r["summary"],
3376                                  "Not Healthy: 1 shares (enc 3-of-10)")
3377             self.failIf(r["results"]["healthy"])
3378             self.failIf(r["results"]["needs-rebalancing"])
3379             self.failIf(r["results"]["recoverable"])
3380         d.addCallback(_got_json_dead)
3381
3382         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3383         def _got_html_corrupt(res):
3384             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3385         d.addCallback(_got_html_corrupt)
3386         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3387         def _got_json_corrupt(res):
3388             r = simplejson.loads(res)
3389             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3390                             r["summary"])
3391             self.failIf(r["results"]["healthy"])
3392             self.failUnless(r["results"]["recoverable"])
3393             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3394             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3395         d.addCallback(_got_json_corrupt)
3396
3397         d.addErrback(self.explain_web_error)
3398         return d
3399
3400     def test_repair_html(self):
3401         self.basedir = "web/Grid/repair_html"
3402         self.set_up_grid()
3403         c0 = self.g.clients[0]
3404         self.uris = {}
3405         DATA = "data" * 100
3406         d = c0.upload(upload.Data(DATA, convergence=""))
3407         def _stash_uri(ur, which):
3408             self.uris[which] = ur.uri
3409         d.addCallback(_stash_uri, "good")
3410         d.addCallback(lambda ign:
3411                       c0.upload(upload.Data(DATA+"1", convergence="")))
3412         d.addCallback(_stash_uri, "sick")
3413         d.addCallback(lambda ign:
3414                       c0.upload(upload.Data(DATA+"2", convergence="")))
3415         d.addCallback(_stash_uri, "dead")
3416         def _stash_mutable_uri(n, which):
3417             self.uris[which] = n.get_uri()
3418             assert isinstance(self.uris[which], str)
3419         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3420         d.addCallback(_stash_mutable_uri, "corrupt")
3421
3422         def _compute_fileurls(ignored):
3423             self.fileurls = {}
3424             for which in self.uris:
3425                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3426         d.addCallback(_compute_fileurls)
3427
3428         def _clobber_shares(ignored):
3429             good_shares = self.find_uri_shares(self.uris["good"])
3430             self.failUnlessReallyEqual(len(good_shares), 10)
3431             sick_shares = self.find_uri_shares(self.uris["sick"])
3432             os.unlink(sick_shares[0][2])
3433             dead_shares = self.find_uri_shares(self.uris["dead"])
3434             for i in range(1, 10):
3435                 os.unlink(dead_shares[i][2])
3436             c_shares = self.find_uri_shares(self.uris["corrupt"])
3437             cso = CorruptShareOptions()
3438             cso.stdout = StringIO()
3439             cso.parseOptions([c_shares[0][2]])
3440             corrupt_share(cso)
3441         d.addCallback(_clobber_shares)
3442
3443         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3444         def _got_html_good(res):
3445             self.failUnless("Healthy" in res, res)
3446             self.failIf("Not Healthy" in res, res)
3447             self.failUnless("No repair necessary" in res, res)
3448         d.addCallback(_got_html_good)
3449
3450         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3451         def _got_html_sick(res):
3452             self.failUnless("Healthy : healthy" in res, res)
3453             self.failIf("Not Healthy" in res, res)
3454             self.failUnless("Repair successful" in res, res)
3455         d.addCallback(_got_html_sick)
3456
3457         # repair of a dead file will fail, of course, but it isn't yet
3458         # clear how this should be reported. Right now it shows up as
3459         # a "410 Gone".
3460         #
3461         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3462         #def _got_html_dead(res):
3463         #    print res
3464         #    self.failUnless("Healthy : healthy" in res, res)
3465         #    self.failIf("Not Healthy" in res, res)
3466         #    self.failUnless("No repair necessary" in res, res)
3467         #d.addCallback(_got_html_dead)
3468
3469         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3470         def _got_html_corrupt(res):
3471             self.failUnless("Healthy : Healthy" in res, res)
3472             self.failIf("Not Healthy" in res, res)
3473             self.failUnless("Repair successful" in res, res)
3474         d.addCallback(_got_html_corrupt)
3475
3476         d.addErrback(self.explain_web_error)
3477         return d
3478
3479     def test_repair_json(self):
3480         self.basedir = "web/Grid/repair_json"
3481         self.set_up_grid()
3482         c0 = self.g.clients[0]
3483         self.uris = {}
3484         DATA = "data" * 100
3485         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3486         def _stash_uri(ur, which):
3487             self.uris[which] = ur.uri
3488         d.addCallback(_stash_uri, "sick")
3489
3490         def _compute_fileurls(ignored):
3491             self.fileurls = {}
3492             for which in self.uris:
3493                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3494         d.addCallback(_compute_fileurls)
3495
3496         def _clobber_shares(ignored):
3497             sick_shares = self.find_uri_shares(self.uris["sick"])
3498             os.unlink(sick_shares[0][2])
3499         d.addCallback(_clobber_shares)
3500
3501         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3502         def _got_json_sick(res):
3503             r = simplejson.loads(res)
3504             self.failUnlessReallyEqual(r["repair-attempted"], True)
3505             self.failUnlessReallyEqual(r["repair-successful"], True)
3506             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3507                                  "Not Healthy: 9 shares (enc 3-of-10)")
3508             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3509             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3510             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3511         d.addCallback(_got_json_sick)
3512
3513         d.addErrback(self.explain_web_error)
3514         return d
3515
3516     def test_unknown(self, immutable=False):
3517         self.basedir = "web/Grid/unknown"
3518         if immutable:
3519             self.basedir = "web/Grid/unknown-immutable"
3520
3521         self.set_up_grid()
3522         c0 = self.g.clients[0]
3523         self.uris = {}
3524         self.fileurls = {}
3525
3526         # the future cap format may contain slashes, which must be tolerated
3527         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3528                                                            safe="")
3529
3530         if immutable:
3531             name = u"future-imm"
3532             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3533             d = c0.create_immutable_dirnode({name: (future_node, {})})
3534         else:
3535             name = u"future"
3536             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3537             d = c0.create_dirnode()
3538
3539         def _stash_root_and_create_file(n):
3540             self.rootnode = n
3541             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3542             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3543             if not immutable:
3544                 return self.rootnode.set_node(name, future_node)
3545         d.addCallback(_stash_root_and_create_file)
3546
3547         # make sure directory listing tolerates unknown nodes
3548         d.addCallback(lambda ign: self.GET(self.rooturl))
3549         def _check_directory_html(res, expected_type_suffix):
3550             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3551                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3552                                  re.DOTALL)
3553             self.failUnless(re.search(pattern, res), res)
3554             # find the More Info link for name, should be relative
3555             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3556             info_url = mo.group(1)
3557             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3558         if immutable:
3559             d.addCallback(_check_directory_html, "-IMM")
3560         else:
3561             d.addCallback(_check_directory_html, "")
3562
3563         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3564         def _check_directory_json(res, expect_rw_uri):
3565             data = simplejson.loads(res)
3566             self.failUnlessEqual(data[0], "dirnode")
3567             f = data[1]["children"][name]
3568             self.failUnlessEqual(f[0], "unknown")
3569             if expect_rw_uri:
3570                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
3571             else:
3572                 self.failIfIn("rw_uri", f[1])
3573             if immutable:
3574                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
3575             else:
3576                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
3577             self.failUnless("metadata" in f[1])
3578         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3579
3580         def _check_info(res, expect_rw_uri, expect_ro_uri):
3581             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3582             if expect_rw_uri:
3583                 self.failUnlessIn(unknown_rwcap, res)
3584             if expect_ro_uri:
3585                 if immutable:
3586                     self.failUnlessIn(unknown_immcap, res)
3587                 else:
3588                     self.failUnlessIn(unknown_rocap, res)
3589             else:
3590                 self.failIfIn(unknown_rocap, res)
3591             self.failIfIn("Raw data as", res)
3592             self.failIfIn("Directory writecap", res)
3593             self.failIfIn("Checker Operations", res)
3594             self.failIfIn("Mutable File Operations", res)
3595             self.failIfIn("Directory Operations", res)
3596
3597         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3598         # why they fail. Possibly related to ticket #922.
3599
3600         d.addCallback(lambda ign: self.GET(expected_info_url))
3601         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3602         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3603         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3604
3605         def _check_json(res, expect_rw_uri):
3606             data = simplejson.loads(res)
3607             self.failUnlessEqual(data[0], "unknown")
3608             if expect_rw_uri:
3609                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
3610             else:
3611                 self.failIfIn("rw_uri", data[1])
3612
3613             if immutable:
3614                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
3615                 self.failUnlessReallyEqual(data[1]["mutable"], False)
3616             elif expect_rw_uri:
3617                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3618                 self.failUnlessReallyEqual(data[1]["mutable"], True)
3619             else:
3620                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3621                 self.failIf("mutable" in data[1], data[1])
3622
3623             # TODO: check metadata contents
3624             self.failUnless("metadata" in data[1])
3625
3626         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3627         d.addCallback(_check_json, expect_rw_uri=not immutable)
3628
3629         # and make sure that a read-only version of the directory can be
3630         # rendered too. This version will not have unknown_rwcap, whether
3631         # or not future_node was immutable.
3632         d.addCallback(lambda ign: self.GET(self.rourl))
3633         if immutable:
3634             d.addCallback(_check_directory_html, "-IMM")
3635         else:
3636             d.addCallback(_check_directory_html, "-RO")
3637
3638         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3639         d.addCallback(_check_directory_json, expect_rw_uri=False)
3640
3641         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3642         d.addCallback(_check_json, expect_rw_uri=False)
3643
3644         # TODO: check that getting t=info from the Info link in the ro directory
3645         # works, and does not include the writecap URI.
3646         return d
3647
3648     def test_immutable_unknown(self):
3649         return self.test_unknown(immutable=True)
3650
3651     def test_mutant_dirnodes_are_omitted(self):
3652         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3653
3654         self.set_up_grid()
3655         c = self.g.clients[0]
3656         nm = c.nodemaker
3657         self.uris = {}
3658         self.fileurls = {}
3659
3660         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3661         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3662         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3663
3664         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3665         # test the dirnode and web layers separately.
3666
3667         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3668         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3669         # When the directory is read, the mutants should be silently disposed of, leaving
3670         # their lonely sibling.
3671         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3672         # because immutable directories don't have a writecap and therefore that field
3673         # isn't (and can't be) decrypted.
3674         # TODO: The field still exists in the netstring. Technically we should check what
3675         # happens if something is put there (_unpack_contents should raise ValueError),
3676         # but that can wait.
3677
3678         lonely_child = nm.create_from_cap(lonely_uri)
3679         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3680         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3681
3682         def _by_hook_or_by_crook():
3683             return True
3684         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3685             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3686
3687         mutant_write_in_ro_child.get_write_uri    = lambda: None
3688         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3689
3690         kids = {u"lonely":      (lonely_child, {}),
3691                 u"ro":          (mutant_ro_child, {}),
3692                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3693                 }
3694         d = c.create_immutable_dirnode(kids)
3695
3696         def _created(dn):
3697             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3698             self.failIf(dn.is_mutable())
3699             self.failUnless(dn.is_readonly())
3700             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3701             self.failIf(hasattr(dn._node, 'get_writekey'))
3702             rep = str(dn)
3703             self.failUnless("RO-IMM" in rep)
3704             cap = dn.get_cap()
3705             self.failUnlessIn("CHK", cap.to_string())
3706             self.cap = cap
3707             self.rootnode = dn
3708             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3709             return download_to_data(dn._node)
3710         d.addCallback(_created)
3711
3712         def _check_data(data):
3713             # Decode the netstring representation of the directory to check that all children
3714             # are present. This is a bit of an abstraction violation, but there's not really
3715             # any other way to do it given that the real DirectoryNode._unpack_contents would
3716             # strip the mutant children out (which is what we're trying to test, later).
3717             position = 0
3718             numkids = 0
3719             while position < len(data):
3720                 entries, position = split_netstring(data, 1, position)
3721                 entry = entries[0]
3722                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3723                 name = name_utf8.decode("utf-8")
3724                 self.failUnless(rwcapdata == "")
3725                 self.failUnless(name in kids)
3726                 (expected_child, ign) = kids[name]
3727                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3728                 numkids += 1
3729
3730             self.failUnlessReallyEqual(numkids, 3)
3731             return self.rootnode.list()
3732         d.addCallback(_check_data)
3733
3734         # Now when we use the real directory listing code, the mutants should be absent.
3735         def _check_kids(children):
3736             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3737             lonely_node, lonely_metadata = children[u"lonely"]
3738
3739             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3740             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3741         d.addCallback(_check_kids)
3742
3743         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3744         d.addCallback(lambda n: n.list())
3745         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3746
3747         # Make sure the lonely child can be listed in HTML...
3748         d.addCallback(lambda ign: self.GET(self.rooturl))
3749         def _check_html(res):
3750             self.failIfIn("URI:SSK", res)
3751             get_lonely = "".join([r'<td>FILE</td>',
3752                                   r'\s+<td>',
3753                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3754                                   r'</td>',
3755                                   r'\s+<td align="right">%d</td>' % len("one"),
3756                                   ])
3757             self.failUnless(re.search(get_lonely, res), res)
3758
3759             # find the More Info link for name, should be relative
3760             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3761             info_url = mo.group(1)
3762             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3763         d.addCallback(_check_html)
3764
3765         # ... and in JSON.
3766         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3767         def _check_json(res):
3768             data = simplejson.loads(res)
3769             self.failUnlessEqual(data[0], "dirnode")
3770             listed_children = data[1]["children"]
3771             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3772             ll_type, ll_data = listed_children[u"lonely"]
3773             self.failUnlessEqual(ll_type, "filenode")
3774             self.failIf("rw_uri" in ll_data)
3775             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
3776         d.addCallback(_check_json)
3777         return d
3778
3779     def test_deep_check(self):
3780         self.basedir = "web/Grid/deep_check"
3781         self.set_up_grid()
3782         c0 = self.g.clients[0]
3783         self.uris = {}
3784         self.fileurls = {}
3785         DATA = "data" * 100
3786         d = c0.create_dirnode()
3787         def _stash_root_and_create_file(n):
3788             self.rootnode = n
3789             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3790             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3791         d.addCallback(_stash_root_and_create_file)
3792         def _stash_uri(fn, which):
3793             self.uris[which] = fn.get_uri()
3794             return fn
3795         d.addCallback(_stash_uri, "good")
3796         d.addCallback(lambda ign:
3797                       self.rootnode.add_file(u"small",
3798                                              upload.Data("literal",
3799                                                         convergence="")))
3800         d.addCallback(_stash_uri, "small")
3801         d.addCallback(lambda ign:
3802                       self.rootnode.add_file(u"sick",
3803                                              upload.Data(DATA+"1",
3804                                                         convergence="")))
3805         d.addCallback(_stash_uri, "sick")
3806
3807         # this tests that deep-check and stream-manifest will ignore
3808         # UnknownNode instances. Hopefully this will also cover deep-stats.
3809         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3810         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3811
3812         def _clobber_shares(ignored):
3813             self.delete_shares_numbered(self.uris["sick"], [0,1])
3814         d.addCallback(_clobber_shares)
3815
3816         # root
3817         # root/good
3818         # root/small
3819         # root/sick
3820         # root/future
3821
3822         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3823         def _done(res):
3824             try:
3825                 units = [simplejson.loads(line)
3826                          for line in res.splitlines()
3827                          if line]
3828             except ValueError:
3829                 print "response is:", res
3830                 print "undecodeable line was '%s'" % line
3831                 raise
3832             self.failUnlessReallyEqual(len(units), 5+1)
3833             # should be parent-first
3834             u0 = units[0]
3835             self.failUnlessEqual(u0["path"], [])
3836             self.failUnlessEqual(u0["type"], "directory")
3837             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3838             u0cr = u0["check-results"]
3839             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3840
3841             ugood = [u for u in units
3842                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3843             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
3844             ugoodcr = ugood["check-results"]
3845             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3846
3847             stats = units[-1]
3848             self.failUnlessEqual(stats["type"], "stats")
3849             s = stats["stats"]
3850             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3851             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3852             self.failUnlessReallyEqual(s["count-directories"], 1)
3853             self.failUnlessReallyEqual(s["count-unknown"], 1)
3854         d.addCallback(_done)
3855
3856         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3857         def _check_manifest(res):
3858             self.failUnless(res.endswith("\n"))
3859             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3860             self.failUnlessReallyEqual(len(units), 5+1)
3861             self.failUnlessEqual(units[-1]["type"], "stats")
3862             first = units[0]
3863             self.failUnlessEqual(first["path"], [])
3864             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
3865             self.failUnlessEqual(first["type"], "directory")
3866             stats = units[-1]["stats"]
3867             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3868             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3869             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3870             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3871             self.failUnlessReallyEqual(stats["count-unknown"], 1)
3872         d.addCallback(_check_manifest)
3873
3874         # now add root/subdir and root/subdir/grandchild, then make subdir
3875         # unrecoverable, then see what happens
3876
3877         d.addCallback(lambda ign:
3878                       self.rootnode.create_subdirectory(u"subdir"))
3879         d.addCallback(_stash_uri, "subdir")
3880         d.addCallback(lambda subdir_node:
3881                       subdir_node.add_file(u"grandchild",
3882                                            upload.Data(DATA+"2",
3883                                                        convergence="")))
3884         d.addCallback(_stash_uri, "grandchild")
3885
3886         d.addCallback(lambda ign:
3887                       self.delete_shares_numbered(self.uris["subdir"],
3888                                                   range(1, 10)))
3889
3890         # root
3891         # root/good
3892         # root/small
3893         # root/sick
3894         # root/future
3895         # root/subdir [unrecoverable]
3896         # root/subdir/grandchild
3897
3898         # how should a streaming-JSON API indicate fatal error?
3899         # answer: emit ERROR: instead of a JSON string
3900
3901         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3902         def _check_broken_manifest(res):
3903             lines = res.splitlines()
3904             error_lines = [i
3905                            for (i,line) in enumerate(lines)
3906                            if line.startswith("ERROR:")]
3907             if not error_lines:
3908                 self.fail("no ERROR: in output: %s" % (res,))
3909             first_error = error_lines[0]
3910             error_line = lines[first_error]
3911             error_msg = lines[first_error+1:]
3912             error_msg_s = "\n".join(error_msg) + "\n"
3913             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3914                               error_line)
3915             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3916             units = [simplejson.loads(line) for line in lines[:first_error]]
3917             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3918             last_unit = units[-1]
3919             self.failUnlessEqual(last_unit["path"], ["subdir"])
3920         d.addCallback(_check_broken_manifest)
3921
3922         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3923         def _check_broken_deepcheck(res):
3924             lines = res.splitlines()
3925             error_lines = [i
3926                            for (i,line) in enumerate(lines)
3927                            if line.startswith("ERROR:")]
3928             if not error_lines:
3929                 self.fail("no ERROR: in output: %s" % (res,))
3930             first_error = error_lines[0]
3931             error_line = lines[first_error]
3932             error_msg = lines[first_error+1:]
3933             error_msg_s = "\n".join(error_msg) + "\n"
3934             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3935                               error_line)
3936             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3937             units = [simplejson.loads(line) for line in lines[:first_error]]
3938             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3939             last_unit = units[-1]
3940             self.failUnlessEqual(last_unit["path"], ["subdir"])
3941             r = last_unit["check-results"]["results"]
3942             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3943             self.failUnlessReallyEqual(r["count-shares-good"], 1)
3944             self.failUnlessReallyEqual(r["recoverable"], False)
3945         d.addCallback(_check_broken_deepcheck)
3946
3947         d.addErrback(self.explain_web_error)
3948         return d
3949
3950     def test_deep_check_and_repair(self):
3951         self.basedir = "web/Grid/deep_check_and_repair"
3952         self.set_up_grid()
3953         c0 = self.g.clients[0]
3954         self.uris = {}
3955         self.fileurls = {}
3956         DATA = "data" * 100
3957         d = c0.create_dirnode()
3958         def _stash_root_and_create_file(n):
3959             self.rootnode = n
3960             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3961             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3962         d.addCallback(_stash_root_and_create_file)
3963         def _stash_uri(fn, which):
3964             self.uris[which] = fn.get_uri()
3965         d.addCallback(_stash_uri, "good")
3966         d.addCallback(lambda ign:
3967                       self.rootnode.add_file(u"small",
3968                                              upload.Data("literal",
3969                                                         convergence="")))
3970         d.addCallback(_stash_uri, "small")
3971         d.addCallback(lambda ign:
3972                       self.rootnode.add_file(u"sick",
3973                                              upload.Data(DATA+"1",
3974                                                         convergence="")))
3975         d.addCallback(_stash_uri, "sick")
3976         #d.addCallback(lambda ign:
3977         #              self.rootnode.add_file(u"dead",
3978         #                                     upload.Data(DATA+"2",
3979         #                                                convergence="")))
3980         #d.addCallback(_stash_uri, "dead")
3981
3982         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3983         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3984         #d.addCallback(_stash_uri, "corrupt")
3985
3986         def _clobber_shares(ignored):
3987             good_shares = self.find_uri_shares(self.uris["good"])
3988             self.failUnlessReallyEqual(len(good_shares), 10)
3989             sick_shares = self.find_uri_shares(self.uris["sick"])
3990             os.unlink(sick_shares[0][2])
3991             #dead_shares = self.find_uri_shares(self.uris["dead"])
3992             #for i in range(1, 10):
3993             #    os.unlink(dead_shares[i][2])
3994
3995             #c_shares = self.find_uri_shares(self.uris["corrupt"])
3996             #cso = CorruptShareOptions()
3997             #cso.stdout = StringIO()
3998             #cso.parseOptions([c_shares[0][2]])
3999             #corrupt_share(cso)
4000         d.addCallback(_clobber_shares)
4001
4002         # root
4003         # root/good   CHK, 10 shares
4004         # root/small  LIT
4005         # root/sick   CHK, 9 shares
4006
4007         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4008         def _done(res):
4009             units = [simplejson.loads(line)
4010                      for line in res.splitlines()
4011                      if line]
4012             self.failUnlessReallyEqual(len(units), 4+1)
4013             # should be parent-first
4014             u0 = units[0]
4015             self.failUnlessEqual(u0["path"], [])
4016             self.failUnlessEqual(u0["type"], "directory")
4017             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4018             u0crr = u0["check-and-repair-results"]
4019             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4020             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4021
4022             ugood = [u for u in units
4023                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4024             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4025             ugoodcrr = ugood["check-and-repair-results"]
4026             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4027             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4028
4029             usick = [u for u in units
4030                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4031             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4032             usickcrr = usick["check-and-repair-results"]
4033             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4034             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4035             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4036             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4037
4038             stats = units[-1]
4039             self.failUnlessEqual(stats["type"], "stats")
4040             s = stats["stats"]
4041             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4042             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4043             self.failUnlessReallyEqual(s["count-directories"], 1)
4044         d.addCallback(_done)
4045
4046         d.addErrback(self.explain_web_error)
4047         return d
4048
4049     def _count_leases(self, ignored, which):
4050         u = self.uris[which]
4051         shares = self.find_uri_shares(u)
4052         lease_counts = []
4053         for shnum, serverid, fn in shares:
4054             sf = get_share_file(fn)
4055             num_leases = len(list(sf.get_leases()))
4056         lease_counts.append( (fn, num_leases) )
4057         return lease_counts
4058
4059     def _assert_leasecount(self, lease_counts, expected):
4060         for (fn, num_leases) in lease_counts:
4061             if num_leases != expected:
4062                 self.fail("expected %d leases, have %d, on %s" %
4063                           (expected, num_leases, fn))
4064
4065     def test_add_lease(self):
4066         self.basedir = "web/Grid/add_lease"
4067         self.set_up_grid(num_clients=2)
4068         c0 = self.g.clients[0]
4069         self.uris = {}
4070         DATA = "data" * 100
4071         d = c0.upload(upload.Data(DATA, convergence=""))
4072         def _stash_uri(ur, which):
4073             self.uris[which] = ur.uri
4074         d.addCallback(_stash_uri, "one")
4075         d.addCallback(lambda ign:
4076                       c0.upload(upload.Data(DATA+"1", convergence="")))
4077         d.addCallback(_stash_uri, "two")
4078         def _stash_mutable_uri(n, which):
4079             self.uris[which] = n.get_uri()
4080             assert isinstance(self.uris[which], str)
4081         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
4082         d.addCallback(_stash_mutable_uri, "mutable")
4083
4084         def _compute_fileurls(ignored):
4085             self.fileurls = {}
4086             for which in self.uris:
4087                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4088         d.addCallback(_compute_fileurls)
4089
4090         d.addCallback(self._count_leases, "one")
4091         d.addCallback(self._assert_leasecount, 1)
4092         d.addCallback(self._count_leases, "two")
4093         d.addCallback(self._assert_leasecount, 1)
4094         d.addCallback(self._count_leases, "mutable")
4095         d.addCallback(self._assert_leasecount, 1)
4096
4097         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4098         def _got_html_good(res):
4099             self.failUnless("Healthy" in res, res)
4100             self.failIf("Not Healthy" in res, res)
4101         d.addCallback(_got_html_good)
4102
4103         d.addCallback(self._count_leases, "one")
4104         d.addCallback(self._assert_leasecount, 1)
4105         d.addCallback(self._count_leases, "two")
4106         d.addCallback(self._assert_leasecount, 1)
4107         d.addCallback(self._count_leases, "mutable")
4108         d.addCallback(self._assert_leasecount, 1)
4109
4110         # this CHECK uses the original client, which uses the same
4111         # lease-secrets, so it will just renew the original lease
4112         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4113         d.addCallback(_got_html_good)
4114
4115         d.addCallback(self._count_leases, "one")
4116         d.addCallback(self._assert_leasecount, 1)
4117         d.addCallback(self._count_leases, "two")
4118         d.addCallback(self._assert_leasecount, 1)
4119         d.addCallback(self._count_leases, "mutable")
4120         d.addCallback(self._assert_leasecount, 1)
4121
4122         # this CHECK uses an alternate client, which adds a second lease
4123         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4124         d.addCallback(_got_html_good)
4125
4126         d.addCallback(self._count_leases, "one")
4127         d.addCallback(self._assert_leasecount, 2)
4128         d.addCallback(self._count_leases, "two")
4129         d.addCallback(self._assert_leasecount, 1)
4130         d.addCallback(self._count_leases, "mutable")
4131         d.addCallback(self._assert_leasecount, 1)
4132
4133         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4134         d.addCallback(_got_html_good)
4135
4136         d.addCallback(self._count_leases, "one")
4137         d.addCallback(self._assert_leasecount, 2)
4138         d.addCallback(self._count_leases, "two")
4139         d.addCallback(self._assert_leasecount, 1)
4140         d.addCallback(self._count_leases, "mutable")
4141         d.addCallback(self._assert_leasecount, 1)
4142
4143         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4144                       clientnum=1)
4145         d.addCallback(_got_html_good)
4146
4147         d.addCallback(self._count_leases, "one")
4148         d.addCallback(self._assert_leasecount, 2)
4149         d.addCallback(self._count_leases, "two")
4150         d.addCallback(self._assert_leasecount, 1)
4151         d.addCallback(self._count_leases, "mutable")
4152         d.addCallback(self._assert_leasecount, 2)
4153
4154         d.addErrback(self.explain_web_error)
4155         return d
4156
4157     def test_deep_add_lease(self):
4158         self.basedir = "web/Grid/deep_add_lease"
4159         self.set_up_grid(num_clients=2)
4160         c0 = self.g.clients[0]
4161         self.uris = {}
4162         self.fileurls = {}
4163         DATA = "data" * 100
4164         d = c0.create_dirnode()
4165         def _stash_root_and_create_file(n):
4166             self.rootnode = n
4167             self.uris["root"] = n.get_uri()
4168             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4169             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4170         d.addCallback(_stash_root_and_create_file)
4171         def _stash_uri(fn, which):
4172             self.uris[which] = fn.get_uri()
4173         d.addCallback(_stash_uri, "one")
4174         d.addCallback(lambda ign:
4175                       self.rootnode.add_file(u"small",
4176                                              upload.Data("literal",
4177                                                         convergence="")))
4178         d.addCallback(_stash_uri, "small")
4179
4180         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4181         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4182         d.addCallback(_stash_uri, "mutable")
4183
4184         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4185         def _done(res):
4186             units = [simplejson.loads(line)
4187                      for line in res.splitlines()
4188                      if line]
4189             # root, one, small, mutable,   stats
4190             self.failUnlessReallyEqual(len(units), 4+1)
4191         d.addCallback(_done)
4192
4193         d.addCallback(self._count_leases, "root")
4194         d.addCallback(self._assert_leasecount, 1)
4195         d.addCallback(self._count_leases, "one")
4196         d.addCallback(self._assert_leasecount, 1)
4197         d.addCallback(self._count_leases, "mutable")
4198         d.addCallback(self._assert_leasecount, 1)
4199
4200         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4201         d.addCallback(_done)
4202
4203         d.addCallback(self._count_leases, "root")
4204         d.addCallback(self._assert_leasecount, 1)
4205         d.addCallback(self._count_leases, "one")
4206         d.addCallback(self._assert_leasecount, 1)
4207         d.addCallback(self._count_leases, "mutable")
4208         d.addCallback(self._assert_leasecount, 1)
4209
4210         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4211                       clientnum=1)
4212         d.addCallback(_done)
4213
4214         d.addCallback(self._count_leases, "root")
4215         d.addCallback(self._assert_leasecount, 2)
4216         d.addCallback(self._count_leases, "one")
4217         d.addCallback(self._assert_leasecount, 2)
4218         d.addCallback(self._count_leases, "mutable")
4219         d.addCallback(self._assert_leasecount, 2)
4220
4221         d.addErrback(self.explain_web_error)
4222         return d
4223
4224
4225     def test_exceptions(self):
4226         self.basedir = "web/Grid/exceptions"
4227         self.set_up_grid(num_clients=1, num_servers=2)
4228         c0 = self.g.clients[0]
4229         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4230         self.fileurls = {}
4231         DATA = "data" * 100
4232         d = c0.create_dirnode()
4233         def _stash_root(n):
4234             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4235             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4236             return n
4237         d.addCallback(_stash_root)
4238         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4239         def _stash_bad(ur):
4240             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4241             self.delete_shares_numbered(ur.uri, range(1,10))
4242
4243             u = uri.from_string(ur.uri)
4244             u.key = testutil.flip_bit(u.key, 0)
4245             baduri = u.to_string()
4246             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4247         d.addCallback(_stash_bad)
4248         d.addCallback(lambda ign: c0.create_dirnode())
4249         def _mangle_dirnode_1share(n):
4250             u = n.get_uri()
4251             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4252             self.fileurls["dir-1share-json"] = url + "?t=json"
4253             self.delete_shares_numbered(u, range(1,10))
4254         d.addCallback(_mangle_dirnode_1share)
4255         d.addCallback(lambda ign: c0.create_dirnode())
4256         def _mangle_dirnode_0share(n):
4257             u = n.get_uri()
4258             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4259             self.fileurls["dir-0share-json"] = url + "?t=json"
4260             self.delete_shares_numbered(u, range(0,10))
4261         d.addCallback(_mangle_dirnode_0share)
4262
4263         # NotEnoughSharesError should be reported sensibly, with a
4264         # text/plain explanation of the problem, and perhaps some
4265         # information on which shares *could* be found.
4266
4267         d.addCallback(lambda ignored:
4268                       self.shouldHTTPError("GET unrecoverable",
4269                                            410, "Gone", "NoSharesError",
4270                                            self.GET, self.fileurls["0shares"]))
4271         def _check_zero_shares(body):
4272             self.failIf("<html>" in body, body)
4273             body = " ".join(body.strip().split())
4274             exp = ("NoSharesError: no shares could be found. "
4275                    "Zero shares usually indicates a corrupt URI, or that "
4276                    "no servers were connected, but it might also indicate "
4277                    "severe corruption. You should perform a filecheck on "
4278                    "this object to learn more. The full error message is: "
4279                    "no shares (need 3). Last failure: None")
4280             self.failUnlessReallyEqual(exp, body)
4281         d.addCallback(_check_zero_shares)
4282
4283
4284         d.addCallback(lambda ignored:
4285                       self.shouldHTTPError("GET 1share",
4286                                            410, "Gone", "NotEnoughSharesError",
4287                                            self.GET, self.fileurls["1share"]))
4288         def _check_one_share(body):
4289             self.failIf("<html>" in body, body)
4290             body = " ".join(body.strip().split())
4291             msgbase = ("NotEnoughSharesError: This indicates that some "
4292                        "servers were unavailable, or that shares have been "
4293                        "lost to server departure, hard drive failure, or disk "
4294                        "corruption. You should perform a filecheck on "
4295                        "this object to learn more. The full error message is:"
4296                        )
4297             msg1 = msgbase + (" ran out of shares:"
4298                               " complete=sh0"
4299                               " pending="
4300                               " overdue= unused= need 3. Last failure: None")
4301             msg2 = msgbase + (" ran out of shares:"
4302                               " complete="
4303                               " pending=Share(sh0-on-xgru5)"
4304                               " overdue= unused= need 3. Last failure: None")
4305             self.failUnless(body == msg1 or body == msg2, body)
4306         d.addCallback(_check_one_share)
4307
4308         d.addCallback(lambda ignored:
4309                       self.shouldHTTPError("GET imaginary",
4310                                            404, "Not Found", None,
4311                                            self.GET, self.fileurls["imaginary"]))
4312         def _missing_child(body):
4313             self.failUnless("No such child: imaginary" in body, body)
4314         d.addCallback(_missing_child)
4315
4316         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4317         def _check_0shares_dir_html(body):
4318             self.failUnless("<html>" in body, body)
4319             # we should see the regular page, but without the child table or
4320             # the dirops forms
4321             body = " ".join(body.strip().split())
4322             self.failUnlessIn('href="?t=info">More info on this directory',
4323                               body)
4324             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4325                    "could not be retrieved, because there were insufficient "
4326                    "good shares. This might indicate that no servers were "
4327                    "connected, insufficient servers were connected, the URI "
4328                    "was corrupt, or that shares have been lost due to server "
4329                    "departure, hard drive failure, or disk corruption. You "
4330                    "should perform a filecheck on this object to learn more.")
4331             self.failUnlessIn(exp, body)
4332             self.failUnlessIn("No upload forms: directory is unreadable", body)
4333         d.addCallback(_check_0shares_dir_html)
4334
4335         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4336         def _check_1shares_dir_html(body):
4337             # at some point, we'll split UnrecoverableFileError into 0-shares
4338             # and some-shares like we did for immutable files (since there
4339             # are different sorts of advice to offer in each case). For now,
4340             # they present the same way.
4341             self.failUnless("<html>" in body, body)
4342             body = " ".join(body.strip().split())
4343             self.failUnlessIn('href="?t=info">More info on this directory',
4344                               body)
4345             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4346                    "could not be retrieved, because there were insufficient "
4347                    "good shares. This might indicate that no servers were "
4348                    "connected, insufficient servers were connected, the URI "
4349                    "was corrupt, or that shares have been lost due to server "
4350                    "departure, hard drive failure, or disk corruption. You "
4351                    "should perform a filecheck on this object to learn more.")
4352             self.failUnlessIn(exp, body)
4353             self.failUnlessIn("No upload forms: directory is unreadable", body)
4354         d.addCallback(_check_1shares_dir_html)
4355
4356         d.addCallback(lambda ignored:
4357                       self.shouldHTTPError("GET dir-0share-json",
4358                                            410, "Gone", "UnrecoverableFileError",
4359                                            self.GET,
4360                                            self.fileurls["dir-0share-json"]))
4361         def _check_unrecoverable_file(body):
4362             self.failIf("<html>" in body, body)
4363             body = " ".join(body.strip().split())
4364             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4365                    "could not be retrieved, because there were insufficient "
4366                    "good shares. This might indicate that no servers were "
4367                    "connected, insufficient servers were connected, the URI "
4368                    "was corrupt, or that shares have been lost due to server "
4369                    "departure, hard drive failure, or disk corruption. You "
4370                    "should perform a filecheck on this object to learn more.")
4371             self.failUnlessReallyEqual(exp, body)
4372         d.addCallback(_check_unrecoverable_file)
4373
4374         d.addCallback(lambda ignored:
4375                       self.shouldHTTPError("GET dir-1share-json",
4376                                            410, "Gone", "UnrecoverableFileError",
4377                                            self.GET,
4378                                            self.fileurls["dir-1share-json"]))
4379         d.addCallback(_check_unrecoverable_file)
4380
4381         d.addCallback(lambda ignored:
4382                       self.shouldHTTPError("GET imaginary",
4383                                            404, "Not Found", None,
4384                                            self.GET, self.fileurls["imaginary"]))
4385
4386         # attach a webapi child that throws a random error, to test how it
4387         # gets rendered.
4388         w = c0.getServiceNamed("webish")
4389         w.root.putChild("ERRORBOOM", ErrorBoom())
4390
4391         # "Accept: */*" :        should get a text/html stack trace
4392         # "Accept: text/plain" : should get a text/plain stack trace
4393         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4394         # no Accept header:      should get a text/html stack trace
4395
4396         d.addCallback(lambda ignored:
4397                       self.shouldHTTPError("GET errorboom_html",
4398                                            500, "Internal Server Error", None,
4399                                            self.GET, "ERRORBOOM",
4400                                            headers={"accept": ["*/*"]}))
4401         def _internal_error_html1(body):
4402             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4403         d.addCallback(_internal_error_html1)
4404
4405         d.addCallback(lambda ignored:
4406                       self.shouldHTTPError("GET errorboom_text",
4407                                            500, "Internal Server Error", None,
4408                                            self.GET, "ERRORBOOM",
4409                                            headers={"accept": ["text/plain"]}))
4410         def _internal_error_text2(body):
4411             self.failIf("<html>" in body, body)
4412             self.failUnless(body.startswith("Traceback "), body)
4413         d.addCallback(_internal_error_text2)
4414
4415         CLI_accepts = "text/plain, application/octet-stream"
4416         d.addCallback(lambda ignored:
4417                       self.shouldHTTPError("GET errorboom_text",
4418                                            500, "Internal Server Error", None,
4419                                            self.GET, "ERRORBOOM",
4420                                            headers={"accept": [CLI_accepts]}))
4421         def _internal_error_text3(body):
4422             self.failIf("<html>" in body, body)
4423             self.failUnless(body.startswith("Traceback "), body)
4424         d.addCallback(_internal_error_text3)
4425
4426         d.addCallback(lambda ignored:
4427                       self.shouldHTTPError("GET errorboom_text",
4428                                            500, "Internal Server Error", None,
4429                                            self.GET, "ERRORBOOM"))
4430         def _internal_error_html4(body):
4431             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4432         d.addCallback(_internal_error_html4)
4433
4434         def _flush_errors(res):
4435             # Trial: please ignore the CompletelyUnhandledError in the logs
4436             self.flushLoggedErrors(CompletelyUnhandledError)
4437             return res
4438         d.addBoth(_flush_errors)
4439
4440         return d
4441
4442 class CompletelyUnhandledError(Exception):
4443     pass
4444 class ErrorBoom(rend.Page):
4445     def beforeRender(self, ctx):
4446         raise CompletelyUnhandledError("whoops")