src/allmydata/test/test_web.py

   1
   2 import os.path, re, urllib, time
   3 import simplejson
   4 from StringIO import StringIO
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11 from nevow import rend
  12 from allmydata import interfaces, uri, webish, dirnode
  13 from allmydata.storage.shares import get_share_file
  14 from allmydata.storage_client import StorageFarmBroker
  15 from allmydata.immutable import upload
  16 from allmydata.immutable.downloader.status import DownloadStatus
  17 from allmydata.dirnode import DirectoryNode
  18 from allmydata.nodemaker import NodeMaker
  19 from allmydata.unknown import UnknownNode
  20 from allmydata.web import status, common
  21 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  22 from allmydata.util import fileutil, base32
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.util.netstring import split_netstring
  25 from allmydata.util.encodingutil import to_str
  26 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  27      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  28 from allmydata.interfaces import IMutableFileNode
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     def _create_lit(self, cap):
  52         return FakeCHKFileNode(cap)
  53     def _create_immutable(self, cap):
  54         return FakeCHKFileNode(cap)
  55     def _create_mutable(self, cap):
  56         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  57     def create_mutable_file(self, contents="", keysize=None):
  58         n = FakeMutableFileNode(None, None, None, None)
  59         return n.create(contents)
  60
  61 class FakeUploader(service.Service):
  62     name = "uploader"
  63     def upload(self, uploadable, history=None):
  64         d = uploadable.get_size()
  65         d.addCallback(lambda size: uploadable.read(size))
  66         def _got_data(datav):
  67             data = "".join(datav)
  68             n = create_chk_filenode(data)
  69             results = upload.UploadResults()
  70             results.uri = n.get_uri()
  71             return results
  72         d.addCallback(_got_data)
  73         return d
  74     def get_helper_info(self):
  75         return (None, False)
  76
  77 def build_one_ds():
  78     ds = DownloadStatus("storage_index", 1234)
  79     now = time.time()
  80
  81     ds.add_segment_request(0, now)
  82     # segnum, when, start,len, decodetime
  83     ds.add_segment_delivery(0, now+1, 0, 100, 0.5)
  84     ds.add_segment_request(1, now+2)
  85     ds.add_segment_error(1, now+3)
  86
  87     e = ds.add_dyhb_sent("serverid_a", now)
  88     e.finished([1,2], now+1)
  89     e = ds.add_dyhb_sent("serverid_b", now+2) # left unfinished
  90
  91     e = ds.add_read_event(0, 120, now)
  92     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
  93     e.finished(now+1)
  94     e = ds.add_read_event(120, 30, now+2) # left unfinished
  95
  96     e = ds.add_request_sent("serverid_a", 1, 100, 20, now)
  97     e.finished(20, now+1)
  98     e = ds.add_request_sent("serverid_a", 1, 120, 30, now+1) # left unfinished
  99
 100     # make sure that add_read_event() can come first too
 101     ds1 = DownloadStatus("storage_index", 1234)
 102     e = ds1.add_read_event(0, 120, now)
 103     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 104     e.finished(now+1)
 105
 106     return ds
 107
 108 class FakeHistory:
 109     _all_upload_status = [upload.UploadStatus()]
 110     _all_download_status = [build_one_ds()]
 111     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 112     _all_publish_statuses = [publish.PublishStatus()]
 113     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 114
 115     def list_all_upload_statuses(self):
 116         return self._all_upload_status
 117     def list_all_download_statuses(self):
 118         return self._all_download_status
 119     def list_all_mapupdate_statuses(self):
 120         return self._all_mapupdate_statuses
 121     def list_all_publish_statuses(self):
 122         return self._all_publish_statuses
 123     def list_all_retrieve_statuses(self):
 124         return self._all_retrieve_statuses
 125     def list_all_helper_statuses(self):
 126         return []
 127
 128 class FakeClient(Client):
 129     def __init__(self):
 130         # don't upcall to Client.__init__, since we only want to initialize a
 131         # minimal subset
 132         service.MultiService.__init__(self)
 133         self.nodeid = "fake_nodeid"
 134         self.nickname = "fake_nickname"
 135         self.introducer_furl = "None"
 136         self.stats_provider = FakeStatsProvider()
 137         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 138         self.helper = None
 139         self.convergence = "some random string"
 140         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 141         self.introducer_client = None
 142         self.history = FakeHistory()
 143         self.uploader = FakeUploader()
 144         self.uploader.setServiceParent(self)
 145         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 146                                        self.uploader, None,
 147                                        None, None)
 148
 149     def startService(self):
 150         return service.MultiService.startService(self)
 151     def stopService(self):
 152         return service.MultiService.stopService(self)
 153
 154     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 155
 156 class WebMixin(object):
 157     def setUp(self):
 158         self.s = FakeClient()
 159         self.s.startService()
 160         self.staticdir = self.mktemp()
 161         self.clock = Clock()
 162         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 163                                       clock=self.clock)
 164         self.ws.setServiceParent(self.s)
 165         self.webish_port = port = self.ws.listener._port.getHost().port
 166         self.webish_url = "http://localhost:%d" % port
 167
 168         l = [ self.s.create_dirnode() for x in range(6) ]
 169         d = defer.DeferredList(l)
 170         def _then(res):
 171             self.public_root = res[0][1]
 172             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 173             self.public_url = "/uri/" + self.public_root.get_uri()
 174             self.private_root = res[1][1]
 175
 176             foo = res[2][1]
 177             self._foo_node = foo
 178             self._foo_uri = foo.get_uri()
 179             self._foo_readonly_uri = foo.get_readonly_uri()
 180             self._foo_verifycap = foo.get_verify_cap().to_string()
 181             # NOTE: we ignore the deferred on all set_uri() calls, because we
 182             # know the fake nodes do these synchronously
 183             self.public_root.set_uri(u"foo", foo.get_uri(),
 184                                      foo.get_readonly_uri())
 185
 186             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 187             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 188             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 189
 190             foo.set_uri(u"empty", res[3][1].get_uri(),
 191                         res[3][1].get_readonly_uri())
 192             sub_uri = res[4][1].get_uri()
 193             self._sub_uri = sub_uri
 194             foo.set_uri(u"sub", sub_uri, sub_uri)
 195             sub = self.s.create_node_from_uri(sub_uri)
 196
 197             _ign, n, blocking_uri = self.makefile(1)
 198             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 199
 200             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 201             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 202             # still think of it as an umlaut
 203             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 204
 205             _ign, n, baz_file = self.makefile(2)
 206             self._baz_file_uri = baz_file
 207             sub.set_uri(u"baz.txt", baz_file, baz_file)
 208
 209             _ign, n, self._bad_file_uri = self.makefile(3)
 210             # this uri should not be downloadable
 211             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 212
 213             rodir = res[5][1]
 214             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 215                                      rodir.get_readonly_uri())
 216             rodir.set_uri(u"nor", baz_file, baz_file)
 217
 218             # public/
 219             # public/foo/
 220             # public/foo/bar.txt
 221             # public/foo/blockingfile
 222             # public/foo/empty/
 223             # public/foo/sub/
 224             # public/foo/sub/baz.txt
 225             # public/reedownlee/
 226             # public/reedownlee/nor
 227             self.NEWFILE_CONTENTS = "newfile contents\n"
 228
 229             return foo.get_metadata_for(u"bar.txt")
 230         d.addCallback(_then)
 231         def _got_metadata(metadata):
 232             self._bar_txt_metadata = metadata
 233         d.addCallback(_got_metadata)
 234         return d
 235
 236     def makefile(self, number):
 237         contents = "contents of file %s\n" % number
 238         n = create_chk_filenode(contents)
 239         return contents, n, n.get_uri()
 240
 241     def tearDown(self):
 242         return self.s.stopService()
 243
 244     def failUnlessIsBarDotTxt(self, res):
 245         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 246
 247     def failUnlessIsBarJSON(self, res):
 248         data = simplejson.loads(res)
 249         self.failUnless(isinstance(data, list))
 250         self.failUnlessEqual(data[0], "filenode")
 251         self.failUnless(isinstance(data[1], dict))
 252         self.failIf(data[1]["mutable"])
 253         self.failIf("rw_uri" in data[1]) # immutable
 254         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 255         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 256         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 257
 258     def failUnlessIsFooJSON(self, res):
 259         data = simplejson.loads(res)
 260         self.failUnless(isinstance(data, list))
 261         self.failUnlessEqual(data[0], "dirnode", res)
 262         self.failUnless(isinstance(data[1], dict))
 263         self.failUnless(data[1]["mutable"])
 264         self.failUnless("rw_uri" in data[1]) # mutable
 265         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 266         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 267         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 268
 269         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 270         self.failUnlessEqual(kidnames,
 271                              [u"bar.txt", u"blockingfile", u"empty",
 272                               u"n\u00fc.txt", u"sub"])
 273         kids = dict( [(unicode(name),value)
 274                       for (name,value)
 275                       in data[1]["children"].iteritems()] )
 276         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 277         self.failUnlessIn("metadata", kids[u"sub"][1])
 278         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 279         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 280         self.failUnlessIn("linkcrtime", tahoe_md)
 281         self.failUnlessIn("linkmotime", tahoe_md)
 282         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 283         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 284         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 285         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 286                                    self._bar_txt_verifycap)
 287         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 288         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 289         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 290                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 291         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 292                                    self._bar_txt_uri)
 293
 294     def GET(self, urlpath, followRedirect=False, return_response=False,
 295             **kwargs):
 296         # if return_response=True, this fires with (data, statuscode,
 297         # respheaders) instead of just data.
 298         assert not isinstance(urlpath, unicode)
 299         url = self.webish_url + urlpath
 300         factory = HTTPClientGETFactory(url, method="GET",
 301                                        followRedirect=followRedirect, **kwargs)
 302         reactor.connectTCP("localhost", self.webish_port, factory)
 303         d = factory.deferred
 304         def _got_data(data):
 305             return (data, factory.status, factory.response_headers)
 306         if return_response:
 307             d.addCallback(_got_data)
 308         return factory.deferred
 309
 310     def HEAD(self, urlpath, return_response=False, **kwargs):
 311         # this requires some surgery, because twisted.web.client doesn't want
 312         # to give us back the response headers.
 313         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 314         reactor.connectTCP("localhost", self.webish_port, factory)
 315         d = factory.deferred
 316         def _got_data(data):
 317             return (data, factory.status, factory.response_headers)
 318         if return_response:
 319             d.addCallback(_got_data)
 320         return factory.deferred
 321
 322     def PUT(self, urlpath, data, **kwargs):
 323         url = self.webish_url + urlpath
 324         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 325
 326     def DELETE(self, urlpath):
 327         url = self.webish_url + urlpath
 328         return client.getPage(url, method="DELETE")
 329
 330     def POST(self, urlpath, followRedirect=False, **fields):
 331         sepbase = "boogabooga"
 332         sep = "--" + sepbase
 333         form = []
 334         form.append(sep)
 335         form.append('Content-Disposition: form-data; name="_charset"')
 336         form.append('')
 337         form.append('UTF-8')
 338         form.append(sep)
 339         for name, value in fields.iteritems():
 340             if isinstance(value, tuple):
 341                 filename, value = value
 342                 form.append('Content-Disposition: form-data; name="%s"; '
 343                             'filename="%s"' % (name, filename.encode("utf-8")))
 344             else:
 345                 form.append('Content-Disposition: form-data; name="%s"' % name)
 346             form.append('')
 347             if isinstance(value, unicode):
 348                 value = value.encode("utf-8")
 349             else:
 350                 value = str(value)
 351             assert isinstance(value, str)
 352             form.append(value)
 353             form.append(sep)
 354         form[-1] += "--"
 355         body = ""
 356         headers = {}
 357         if fields:
 358             body = "\r\n".join(form) + "\r\n"
 359             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 360         return self.POST2(urlpath, body, headers, followRedirect)
 361
 362     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 363         url = self.webish_url + urlpath
 364         return client.getPage(url, method="POST", postdata=body,
 365                               headers=headers, followRedirect=followRedirect)
 366
 367     def shouldFail(self, res, expected_failure, which,
 368                    substring=None, response_substring=None):
 369         if isinstance(res, failure.Failure):
 370             res.trap(expected_failure)
 371             if substring:
 372                 self.failUnless(substring in str(res),
 373                                 "substring '%s' not in '%s'"
 374                                 % (substring, str(res)))
 375             if response_substring:
 376                 self.failUnless(response_substring in res.value.response,
 377                                 "response substring '%s' not in '%s'"
 378                                 % (response_substring, res.value.response))
 379         else:
 380             self.fail("%s was supposed to raise %s, not get '%s'" %
 381                       (which, expected_failure, res))
 382
 383     def shouldFail2(self, expected_failure, which, substring,
 384                     response_substring,
 385                     callable, *args, **kwargs):
 386         assert substring is None or isinstance(substring, str)
 387         assert response_substring is None or isinstance(response_substring, str)
 388         d = defer.maybeDeferred(callable, *args, **kwargs)
 389         def done(res):
 390             if isinstance(res, failure.Failure):
 391                 res.trap(expected_failure)
 392                 if substring:
 393                     self.failUnless(substring in str(res),
 394                                     "%s: substring '%s' not in '%s'"
 395                                     % (which, substring, str(res)))
 396                 if response_substring:
 397                     self.failUnless(response_substring in res.value.response,
 398                                     "%s: response substring '%s' not in '%s'"
 399                                     % (which,
 400                                        response_substring, res.value.response))
 401             else:
 402                 self.fail("%s was supposed to raise %s, not get '%s'" %
 403                           (which, expected_failure, res))
 404         d.addBoth(done)
 405         return d
 406
 407     def should404(self, res, which):
 408         if isinstance(res, failure.Failure):
 409             res.trap(error.Error)
 410             self.failUnlessReallyEqual(res.value.status, "404")
 411         else:
 412             self.fail("%s was supposed to Error(404), not get '%s'" %
 413                       (which, res))
 414
 415     def should302(self, res, which):
 416         if isinstance(res, failure.Failure):
 417             res.trap(error.Error)
 418             self.failUnlessReallyEqual(res.value.status, "302")
 419         else:
 420             self.fail("%s was supposed to Error(302), not get '%s'" %
 421                         (which, res))
 422
 423
 424 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 425     def test_create(self):
 426         pass
 427
 428     def test_welcome(self):
 429         d = self.GET("/")
 430         def _check(res):
 431             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 432
 433             self.s.basedir = 'web/test_welcome'
 434             fileutil.make_dirs("web/test_welcome")
 435             fileutil.make_dirs("web/test_welcome/private")
 436             return self.GET("/")
 437         d.addCallback(_check)
 438         return d
 439
 440     def test_provisioning(self):
 441         d = self.GET("/provisioning/")
 442         def _check(res):
 443             self.failUnless('Provisioning Tool' in res)
 444             fields = {'filled': True,
 445                       "num_users": int(50e3),
 446                       "files_per_user": 1000,
 447                       "space_per_user": int(1e9),
 448                       "sharing_ratio": 1.0,
 449                       "encoding_parameters": "3-of-10-5",
 450                       "num_servers": 30,
 451                       "ownership_mode": "A",
 452                       "download_rate": 100,
 453                       "upload_rate": 10,
 454                       "delete_rate": 10,
 455                       "lease_timer": 7,
 456                       }
 457             return self.POST("/provisioning/", **fields)
 458
 459         d.addCallback(_check)
 460         def _check2(res):
 461             self.failUnless('Provisioning Tool' in res)
 462             self.failUnless("Share space consumed: 167.01TB" in res)
 463
 464             fields = {'filled': True,
 465                       "num_users": int(50e6),
 466                       "files_per_user": 1000,
 467                       "space_per_user": int(5e9),
 468                       "sharing_ratio": 1.0,
 469                       "encoding_parameters": "25-of-100-50",
 470                       "num_servers": 30000,
 471                       "ownership_mode": "E",
 472                       "drive_failure_model": "U",
 473                       "drive_size": 1000,
 474                       "download_rate": 1000,
 475                       "upload_rate": 100,
 476                       "delete_rate": 100,
 477                       "lease_timer": 7,
 478                       }
 479             return self.POST("/provisioning/", **fields)
 480         d.addCallback(_check2)
 481         def _check3(res):
 482             self.failUnless("Share space consumed: huge!" in res)
 483             fields = {'filled': True}
 484             return self.POST("/provisioning/", **fields)
 485         d.addCallback(_check3)
 486         def _check4(res):
 487             self.failUnless("Share space consumed:" in res)
 488         d.addCallback(_check4)
 489         return d
 490
 491     def test_reliability_tool(self):
 492         try:
 493             from allmydata import reliability
 494             _hush_pyflakes = reliability
 495             del _hush_pyflakes
 496         except:
 497             raise unittest.SkipTest("reliability tool requires NumPy")
 498
 499         d = self.GET("/reliability/")
 500         def _check(res):
 501             self.failUnless('Reliability Tool' in res)
 502             fields = {'drive_lifetime': "8Y",
 503                       "k": "3",
 504                       "R": "7",
 505                       "N": "10",
 506                       "delta": "100000",
 507                       "check_period": "1M",
 508                       "report_period": "3M",
 509                       "report_span": "5Y",
 510                       }
 511             return self.POST("/reliability/", **fields)
 512
 513         d.addCallback(_check)
 514         def _check2(res):
 515             self.failUnless('Reliability Tool' in res)
 516             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 517             self.failUnless(re.search(r, res), res)
 518         d.addCallback(_check2)
 519         return d
 520
 521     def test_status(self):
 522         h = self.s.get_history()
 523         dl_num = h.list_all_download_statuses()[0].get_counter()
 524         ul_num = h.list_all_upload_statuses()[0].get_counter()
 525         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 526         pub_num = h.list_all_publish_statuses()[0].get_counter()
 527         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 528         d = self.GET("/status", followRedirect=True)
 529         def _check(res):
 530             self.failUnless('Upload and Download Status' in res, res)
 531             self.failUnless('"down-%d"' % dl_num in res, res)
 532             self.failUnless('"up-%d"' % ul_num in res, res)
 533             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 534             self.failUnless('"publish-%d"' % pub_num in res, res)
 535             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 536         d.addCallback(_check)
 537         d.addCallback(lambda res: self.GET("/status/?t=json"))
 538         def _check_json(res):
 539             data = simplejson.loads(res)
 540             self.failUnless(isinstance(data, dict))
 541             #active = data["active"]
 542             # TODO: test more. We need a way to fake an active operation
 543             # here.
 544         d.addCallback(_check_json)
 545
 546         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 547         def _check_dl(res):
 548             self.failUnless("File Download Status" in res, res)
 549         d.addCallback(_check_dl)
 550         d.addCallback(lambda res: self.GET("/status/down-%d?t=json" % dl_num))
 551         def _check_dl_json(res):
 552             data = simplejson.loads(res)
 553             self.failUnless(isinstance(data, dict))
 554         d.addCallback(_check_dl_json)
 555         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 556         def _check_ul(res):
 557             self.failUnless("File Upload Status" in res, res)
 558         d.addCallback(_check_ul)
 559         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 560         def _check_mapupdate(res):
 561             self.failUnless("Mutable File Servermap Update Status" in res, res)
 562         d.addCallback(_check_mapupdate)
 563         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 564         def _check_publish(res):
 565             self.failUnless("Mutable File Publish Status" in res, res)
 566         d.addCallback(_check_publish)
 567         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 568         def _check_retrieve(res):
 569             self.failUnless("Mutable File Retrieve Status" in res, res)
 570         d.addCallback(_check_retrieve)
 571
 572         return d
 573
 574     def test_status_numbers(self):
 575         drrm = status.DownloadResultsRendererMixin()
 576         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 577         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 578         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 579         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 580         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 581         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 582         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 583         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 584         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 585
 586         urrm = status.UploadResultsRendererMixin()
 587         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 588         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 589         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 590         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 591         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 592         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 593         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 594         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 595         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 596
 597     def test_GET_FILEURL(self):
 598         d = self.GET(self.public_url + "/foo/bar.txt")
 599         d.addCallback(self.failUnlessIsBarDotTxt)
 600         return d
 601
 602     def test_GET_FILEURL_range(self):
 603         headers = {"range": "bytes=1-10"}
 604         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 605                      return_response=True)
 606         def _got((res, status, headers)):
 607             self.failUnlessReallyEqual(int(status), 206)
 608             self.failUnless(headers.has_key("content-range"))
 609             self.failUnlessReallyEqual(headers["content-range"][0],
 610                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 611             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 612         d.addCallback(_got)
 613         return d
 614
 615     def test_GET_FILEURL_partial_range(self):
 616         headers = {"range": "bytes=5-"}
 617         length  = len(self.BAR_CONTENTS)
 618         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 619                      return_response=True)
 620         def _got((res, status, headers)):
 621             self.failUnlessReallyEqual(int(status), 206)
 622             self.failUnless(headers.has_key("content-range"))
 623             self.failUnlessReallyEqual(headers["content-range"][0],
 624                                        "bytes 5-%d/%d" % (length-1, length))
 625             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 626         d.addCallback(_got)
 627         return d
 628
 629     def test_GET_FILEURL_partial_end_range(self):
 630         headers = {"range": "bytes=-5"}
 631         length  = len(self.BAR_CONTENTS)
 632         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 633                      return_response=True)
 634         def _got((res, status, headers)):
 635             self.failUnlessReallyEqual(int(status), 206)
 636             self.failUnless(headers.has_key("content-range"))
 637             self.failUnlessReallyEqual(headers["content-range"][0],
 638                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 639             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 640         d.addCallback(_got)
 641         return d
 642
 643     def test_GET_FILEURL_partial_range_overrun(self):
 644         headers = {"range": "bytes=100-200"}
 645         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 646                              "416 Requested Range not satisfiable",
 647                              "First beyond end of file",
 648                              self.GET, self.public_url + "/foo/bar.txt",
 649                              headers=headers)
 650         return d
 651
 652     def test_HEAD_FILEURL_range(self):
 653         headers = {"range": "bytes=1-10"}
 654         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 655                      return_response=True)
 656         def _got((res, status, headers)):
 657             self.failUnlessReallyEqual(res, "")
 658             self.failUnlessReallyEqual(int(status), 206)
 659             self.failUnless(headers.has_key("content-range"))
 660             self.failUnlessReallyEqual(headers["content-range"][0],
 661                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 662         d.addCallback(_got)
 663         return d
 664
 665     def test_HEAD_FILEURL_partial_range(self):
 666         headers = {"range": "bytes=5-"}
 667         length  = len(self.BAR_CONTENTS)
 668         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 669                      return_response=True)
 670         def _got((res, status, headers)):
 671             self.failUnlessReallyEqual(int(status), 206)
 672             self.failUnless(headers.has_key("content-range"))
 673             self.failUnlessReallyEqual(headers["content-range"][0],
 674                                        "bytes 5-%d/%d" % (length-1, length))
 675         d.addCallback(_got)
 676         return d
 677
 678     def test_HEAD_FILEURL_partial_end_range(self):
 679         headers = {"range": "bytes=-5"}
 680         length  = len(self.BAR_CONTENTS)
 681         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 682                      return_response=True)
 683         def _got((res, status, headers)):
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range_overrun(self):
 692         headers = {"range": "bytes=100-200"}
 693         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 694                              "416 Requested Range not satisfiable",
 695                              "",
 696                              self.HEAD, self.public_url + "/foo/bar.txt",
 697                              headers=headers)
 698         return d
 699
 700     def test_GET_FILEURL_range_bad(self):
 701         headers = {"range": "BOGUS=fizbop-quarnak"}
 702         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 703                      return_response=True)
 704         def _got((res, status, headers)):
 705             self.failUnlessReallyEqual(int(status), 200)
 706             self.failUnless(not headers.has_key("content-range"))
 707             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 708         d.addCallback(_got)
 709         return d
 710
 711     def test_HEAD_FILEURL(self):
 712         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 713         def _got((res, status, headers)):
 714             self.failUnlessReallyEqual(res, "")
 715             self.failUnlessReallyEqual(headers["content-length"][0],
 716                                        str(len(self.BAR_CONTENTS)))
 717             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 718         d.addCallback(_got)
 719         return d
 720
 721     def test_GET_FILEURL_named(self):
 722         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 723         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 724         d = self.GET(base + "/@@name=/blah.txt")
 725         d.addCallback(self.failUnlessIsBarDotTxt)
 726         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 727         d.addCallback(self.failUnlessIsBarDotTxt)
 728         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 729         d.addCallback(self.failUnlessIsBarDotTxt)
 730         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 731         d.addCallback(self.failUnlessIsBarDotTxt)
 732         save_url = base + "?save=true&filename=blah.txt"
 733         d.addCallback(lambda res: self.GET(save_url))
 734         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 735         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 736         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 737         u_url = base + "?save=true&filename=" + u_fn_e
 738         d.addCallback(lambda res: self.GET(u_url))
 739         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 740         return d
 741
 742     def test_PUT_FILEURL_named_bad(self):
 743         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 744         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 745                              "400 Bad Request",
 746                              "/file can only be used with GET or HEAD",
 747                              self.PUT, base + "/@@name=/blah.txt", "")
 748         return d
 749
 750     def test_GET_DIRURL_named_bad(self):
 751         base = "/file/%s" % urllib.quote(self._foo_uri)
 752         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 753                              "400 Bad Request",
 754                              "is not a file-cap",
 755                              self.GET, base + "/@@name=/blah.txt")
 756         return d
 757
 758     def test_GET_slash_file_bad(self):
 759         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 760                              "404 Not Found",
 761                              "/file must be followed by a file-cap and a name",
 762                              self.GET, "/file")
 763         return d
 764
 765     def test_GET_unhandled_URI_named(self):
 766         contents, n, newuri = self.makefile(12)
 767         verifier_cap = n.get_verify_cap().to_string()
 768         base = "/file/%s" % urllib.quote(verifier_cap)
 769         # client.create_node_from_uri() can't handle verify-caps
 770         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 771                              "400 Bad Request", "is not a file-cap",
 772                              self.GET, base)
 773         return d
 774
 775     def test_GET_unhandled_URI(self):
 776         contents, n, newuri = self.makefile(12)
 777         verifier_cap = n.get_verify_cap().to_string()
 778         base = "/uri/%s" % urllib.quote(verifier_cap)
 779         # client.create_node_from_uri() can't handle verify-caps
 780         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 781                              "400 Bad Request",
 782                              "GET unknown URI type: can only do t=info",
 783                              self.GET, base)
 784         return d
 785
 786     def test_GET_FILE_URI(self):
 787         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 788         d = self.GET(base)
 789         d.addCallback(self.failUnlessIsBarDotTxt)
 790         return d
 791
 792     def test_GET_FILE_URI_badchild(self):
 793         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 794         errmsg = "Files have no children, certainly not named 'boguschild'"
 795         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 796                              "400 Bad Request", errmsg,
 797                              self.GET, base)
 798         return d
 799
 800     def test_PUT_FILE_URI_badchild(self):
 801         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 802         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 803         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 804                              "400 Bad Request", errmsg,
 805                              self.PUT, base, "")
 806         return d
 807
 808     # TODO: version of this with a Unicode filename
 809     def test_GET_FILEURL_save(self):
 810         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 811                      return_response=True)
 812         def _got((res, statuscode, headers)):
 813             content_disposition = headers["content-disposition"][0]
 814             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 815             self.failUnlessIsBarDotTxt(res)
 816         d.addCallback(_got)
 817         return d
 818
 819     def test_GET_FILEURL_missing(self):
 820         d = self.GET(self.public_url + "/foo/missing")
 821         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 822         return d
 823
 824     def test_PUT_overwrite_only_files(self):
 825         # create a directory, put a file in that directory.
 826         contents, n, filecap = self.makefile(8)
 827         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 828         d.addCallback(lambda res:
 829             self.PUT(self.public_url + "/foo/dir/file1.txt",
 830                      self.NEWFILE_CONTENTS))
 831         # try to overwrite the file with replace=only-files
 832         # (this should work)
 833         d.addCallback(lambda res:
 834             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 835                      filecap))
 836         d.addCallback(lambda res:
 837             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 838                  "There was already a child by that name, and you asked me "
 839                  "to not replace it",
 840                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 841                  filecap))
 842         return d
 843
 844     def test_PUT_NEWFILEURL(self):
 845         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 846         # TODO: we lose the response code, so we can't check this
 847         #self.failUnlessReallyEqual(responsecode, 201)
 848         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 849         d.addCallback(lambda res:
 850                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 851                                                       self.NEWFILE_CONTENTS))
 852         return d
 853
 854     def test_PUT_NEWFILEURL_not_mutable(self):
 855         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 856                      self.NEWFILE_CONTENTS)
 857         # TODO: we lose the response code, so we can't check this
 858         #self.failUnlessReallyEqual(responsecode, 201)
 859         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 860         d.addCallback(lambda res:
 861                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 862                                                       self.NEWFILE_CONTENTS))
 863         return d
 864
 865     def test_PUT_NEWFILEURL_range_bad(self):
 866         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 867         target = self.public_url + "/foo/new.txt"
 868         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 869                              "501 Not Implemented",
 870                              "Content-Range in PUT not yet supported",
 871                              # (and certainly not for immutable files)
 872                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 873                              headers=headers)
 874         d.addCallback(lambda res:
 875                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 876         return d
 877
 878     def test_PUT_NEWFILEURL_mutable(self):
 879         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 880                      self.NEWFILE_CONTENTS)
 881         # TODO: we lose the response code, so we can't check this
 882         #self.failUnlessReallyEqual(responsecode, 201)
 883         def _check_uri(res):
 884             u = uri.from_string_mutable_filenode(res)
 885             self.failUnless(u.is_mutable())
 886             self.failIf(u.is_readonly())
 887             return res
 888         d.addCallback(_check_uri)
 889         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 890         d.addCallback(lambda res:
 891                       self.failUnlessMutableChildContentsAre(self._foo_node,
 892                                                              u"new.txt",
 893                                                              self.NEWFILE_CONTENTS))
 894         return d
 895
 896     def test_PUT_NEWFILEURL_mutable_toobig(self):
 897         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 898                              "413 Request Entity Too Large",
 899                              "SDMF is limited to one segment, and 10001 > 10000",
 900                              self.PUT,
 901                              self.public_url + "/foo/new.txt?mutable=true",
 902                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 903         return d
 904
 905     def test_PUT_NEWFILEURL_replace(self):
 906         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 907         # TODO: we lose the response code, so we can't check this
 908         #self.failUnlessReallyEqual(responsecode, 200)
 909         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 910         d.addCallback(lambda res:
 911                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 912                                                       self.NEWFILE_CONTENTS))
 913         return d
 914
 915     def test_PUT_NEWFILEURL_bad_t(self):
 916         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 917                              "PUT to a file: bad t=bogus",
 918                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 919                              "contents")
 920         return d
 921
 922     def test_PUT_NEWFILEURL_no_replace(self):
 923         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 924                      self.NEWFILE_CONTENTS)
 925         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 926                   "409 Conflict",
 927                   "There was already a child by that name, and you asked me "
 928                   "to not replace it")
 929         return d
 930
 931     def test_PUT_NEWFILEURL_mkdirs(self):
 932         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 933         fn = self._foo_node
 934         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 935         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 936         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 937         d.addCallback(lambda res:
 938                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 939                                                       self.NEWFILE_CONTENTS))
 940         return d
 941
 942     def test_PUT_NEWFILEURL_blocked(self):
 943         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 944                      self.NEWFILE_CONTENTS)
 945         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 946                   "409 Conflict",
 947                   "Unable to create directory 'blockingfile': a file was in the way")
 948         return d
 949
 950     def test_PUT_NEWFILEURL_emptyname(self):
 951         # an empty pathname component (i.e. a double-slash) is disallowed
 952         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 953                              "400 Bad Request",
 954                              "The webapi does not allow empty pathname components",
 955                              self.PUT, self.public_url + "/foo//new.txt", "")
 956         return d
 957
 958     def test_DELETE_FILEURL(self):
 959         d = self.DELETE(self.public_url + "/foo/bar.txt")
 960         d.addCallback(lambda res:
 961                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 962         return d
 963
 964     def test_DELETE_FILEURL_missing(self):
 965         d = self.DELETE(self.public_url + "/foo/missing")
 966         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 967         return d
 968
 969     def test_DELETE_FILEURL_missing2(self):
 970         d = self.DELETE(self.public_url + "/missing/missing")
 971         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 972         return d
 973
 974     def failUnlessHasBarDotTxtMetadata(self, res):
 975         data = simplejson.loads(res)
 976         self.failUnless(isinstance(data, list))
 977         self.failUnlessIn("metadata", data[1])
 978         self.failUnlessIn("tahoe", data[1]["metadata"])
 979         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
 980         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
 981         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
 982                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 983
 984     def test_GET_FILEURL_json(self):
 985         # twisted.web.http.parse_qs ignores any query args without an '=', so
 986         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 987         # instead. This may make it tricky to emulate the S3 interface
 988         # completely.
 989         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 990         def _check1(data):
 991             self.failUnlessIsBarJSON(data)
 992             self.failUnlessHasBarDotTxtMetadata(data)
 993             return
 994         d.addCallback(_check1)
 995         return d
 996
 997     def test_GET_FILEURL_json_missing(self):
 998         d = self.GET(self.public_url + "/foo/missing?json")
 999         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1000         return d
1001
1002     def test_GET_FILEURL_uri(self):
1003         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1004         def _check(res):
1005             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1006         d.addCallback(_check)
1007         d.addCallback(lambda res:
1008                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1009         def _check2(res):
1010             # for now, for files, uris and readonly-uris are the same
1011             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1012         d.addCallback(_check2)
1013         return d
1014
1015     def test_GET_FILEURL_badtype(self):
1016         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1017                                  "bad t=bogus",
1018                                  self.GET,
1019                                  self.public_url + "/foo/bar.txt?t=bogus")
1020         return d
1021
1022     def test_CSS_FILE(self):
1023         d = self.GET("/tahoe_css", followRedirect=True)
1024         def _check(res):
1025             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1026             self.failUnless(CSS_STYLE.search(res), res)
1027         d.addCallback(_check)
1028         return d
1029
1030     def test_GET_FILEURL_uri_missing(self):
1031         d = self.GET(self.public_url + "/foo/missing?t=uri")
1032         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1033         return d
1034
1035     def test_GET_DIRECTORY_html_banner(self):
1036         d = self.GET(self.public_url + "/foo", followRedirect=True)
1037         def _check(res):
1038             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1039         d.addCallback(_check)
1040         return d
1041
1042     def test_GET_DIRURL(self):
1043         # the addSlash means we get a redirect here
1044         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1045         ROOT = "../../.."
1046         d = self.GET(self.public_url + "/foo", followRedirect=True)
1047         def _check(res):
1048             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1049                             in res, res)
1050             # the FILE reference points to a URI, but it should end in bar.txt
1051             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1052                        (ROOT, urllib.quote(self._bar_txt_uri)))
1053             get_bar = "".join([r'<td>FILE</td>',
1054                                r'\s+<td>',
1055                                r'<a href="%s">bar.txt</a>' % bar_url,
1056                                r'</td>',
1057                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
1058                                ])
1059             self.failUnless(re.search(get_bar, res), res)
1060             for line in res.split("\n"):
1061                 # find the line that contains the delete button for bar.txt
1062                 if ("form action" in line and
1063                     'value="delete"' in line and
1064                     'value="bar.txt"' in line):
1065                     # the form target should use a relative URL
1066                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1067                     self.failUnless(('action="%s"' % foo_url) in line, line)
1068                     # and the when_done= should too
1069                     #done_url = urllib.quote(???)
1070                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1071                     #                in line, line)
1072                     break
1073             else:
1074                 self.fail("unable to find delete-bar.txt line", res)
1075
1076             # the DIR reference just points to a URI
1077             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1078             get_sub = ((r'<td>DIR</td>')
1079                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1080             self.failUnless(re.search(get_sub, res), res)
1081         d.addCallback(_check)
1082
1083         # look at a readonly directory
1084         d.addCallback(lambda res:
1085                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1086         def _check2(res):
1087             self.failUnless("(read-only)" in res, res)
1088             self.failIf("Upload a file" in res, res)
1089         d.addCallback(_check2)
1090
1091         # and at a directory that contains a readonly directory
1092         d.addCallback(lambda res:
1093                       self.GET(self.public_url, followRedirect=True))
1094         def _check3(res):
1095             self.failUnless(re.search('<td>DIR-RO</td>'
1096                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1097         d.addCallback(_check3)
1098
1099         # and an empty directory
1100         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1101         def _check4(res):
1102             self.failUnless("directory is empty" in res, res)
1103             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1104             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1105         d.addCallback(_check4)
1106
1107         # and at a literal directory
1108         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1109         d.addCallback(lambda res:
1110                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1111         def _check5(res):
1112             self.failUnless('(immutable)' in res, res)
1113             self.failUnless(re.search('<td>FILE</td>'
1114                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1115         d.addCallback(_check5)
1116         return d
1117
1118     def test_GET_DIRURL_badtype(self):
1119         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1120                                  400, "Bad Request",
1121                                  "bad t=bogus",
1122                                  self.GET,
1123                                  self.public_url + "/foo?t=bogus")
1124         return d
1125
1126     def test_GET_DIRURL_json(self):
1127         d = self.GET(self.public_url + "/foo?t=json")
1128         d.addCallback(self.failUnlessIsFooJSON)
1129         return d
1130
1131
1132     def test_POST_DIRURL_manifest_no_ophandle(self):
1133         d = self.shouldFail2(error.Error,
1134                              "test_POST_DIRURL_manifest_no_ophandle",
1135                              "400 Bad Request",
1136                              "slow operation requires ophandle=",
1137                              self.POST, self.public_url, t="start-manifest")
1138         return d
1139
1140     def test_POST_DIRURL_manifest(self):
1141         d = defer.succeed(None)
1142         def getman(ignored, output):
1143             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1144                           followRedirect=True)
1145             d.addCallback(self.wait_for_operation, "125")
1146             d.addCallback(self.get_operation_results, "125", output)
1147             return d
1148         d.addCallback(getman, None)
1149         def _got_html(manifest):
1150             self.failUnless("Manifest of SI=" in manifest)
1151             self.failUnless("<td>sub</td>" in manifest)
1152             self.failUnless(self._sub_uri in manifest)
1153             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1154         d.addCallback(_got_html)
1155
1156         # both t=status and unadorned GET should be identical
1157         d.addCallback(lambda res: self.GET("/operations/125"))
1158         d.addCallback(_got_html)
1159
1160         d.addCallback(getman, "html")
1161         d.addCallback(_got_html)
1162         d.addCallback(getman, "text")
1163         def _got_text(manifest):
1164             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1165             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1166         d.addCallback(_got_text)
1167         d.addCallback(getman, "JSON")
1168         def _got_json(res):
1169             data = res["manifest"]
1170             got = {}
1171             for (path_list, cap) in data:
1172                 got[tuple(path_list)] = cap
1173             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1174             self.failUnless((u"sub",u"baz.txt") in got)
1175             self.failUnless("finished" in res)
1176             self.failUnless("origin" in res)
1177             self.failUnless("storage-index" in res)
1178             self.failUnless("verifycaps" in res)
1179             self.failUnless("stats" in res)
1180         d.addCallback(_got_json)
1181         return d
1182
1183     def test_POST_DIRURL_deepsize_no_ophandle(self):
1184         d = self.shouldFail2(error.Error,
1185                              "test_POST_DIRURL_deepsize_no_ophandle",
1186                              "400 Bad Request",
1187                              "slow operation requires ophandle=",
1188                              self.POST, self.public_url, t="start-deep-size")
1189         return d
1190
1191     def test_POST_DIRURL_deepsize(self):
1192         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1193                       followRedirect=True)
1194         d.addCallback(self.wait_for_operation, "126")
1195         d.addCallback(self.get_operation_results, "126", "json")
1196         def _got_json(data):
1197             self.failUnlessReallyEqual(data["finished"], True)
1198             size = data["size"]
1199             self.failUnless(size > 1000)
1200         d.addCallback(_got_json)
1201         d.addCallback(self.get_operation_results, "126", "text")
1202         def _got_text(res):
1203             mo = re.search(r'^size: (\d+)$', res, re.M)
1204             self.failUnless(mo, res)
1205             size = int(mo.group(1))
1206             # with directories, the size varies.
1207             self.failUnless(size > 1000)
1208         d.addCallback(_got_text)
1209         return d
1210
1211     def test_POST_DIRURL_deepstats_no_ophandle(self):
1212         d = self.shouldFail2(error.Error,
1213                              "test_POST_DIRURL_deepstats_no_ophandle",
1214                              "400 Bad Request",
1215                              "slow operation requires ophandle=",
1216                              self.POST, self.public_url, t="start-deep-stats")
1217         return d
1218
1219     def test_POST_DIRURL_deepstats(self):
1220         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1221                       followRedirect=True)
1222         d.addCallback(self.wait_for_operation, "127")
1223         d.addCallback(self.get_operation_results, "127", "json")
1224         def _got_json(stats):
1225             expected = {"count-immutable-files": 3,
1226                         "count-mutable-files": 0,
1227                         "count-literal-files": 0,
1228                         "count-files": 3,
1229                         "count-directories": 3,
1230                         "size-immutable-files": 57,
1231                         "size-literal-files": 0,
1232                         #"size-directories": 1912, # varies
1233                         #"largest-directory": 1590,
1234                         "largest-directory-children": 5,
1235                         "largest-immutable-file": 19,
1236                         }
1237             for k,v in expected.iteritems():
1238                 self.failUnlessReallyEqual(stats[k], v,
1239                                            "stats[%s] was %s, not %s" %
1240                                            (k, stats[k], v))
1241             self.failUnlessReallyEqual(stats["size-files-histogram"],
1242                                        [ [11, 31, 3] ])
1243         d.addCallback(_got_json)
1244         return d
1245
1246     def test_POST_DIRURL_stream_manifest(self):
1247         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1248         def _check(res):
1249             self.failUnless(res.endswith("\n"))
1250             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1251             self.failUnlessReallyEqual(len(units), 7)
1252             self.failUnlessEqual(units[-1]["type"], "stats")
1253             first = units[0]
1254             self.failUnlessEqual(first["path"], [])
1255             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1256             self.failUnlessEqual(first["type"], "directory")
1257             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1258             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1259             self.failIfEqual(baz["storage-index"], None)
1260             self.failIfEqual(baz["verifycap"], None)
1261             self.failIfEqual(baz["repaircap"], None)
1262             return
1263         d.addCallback(_check)
1264         return d
1265
1266     def test_GET_DIRURL_uri(self):
1267         d = self.GET(self.public_url + "/foo?t=uri")
1268         def _check(res):
1269             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1270         d.addCallback(_check)
1271         return d
1272
1273     def test_GET_DIRURL_readonly_uri(self):
1274         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1275         def _check(res):
1276             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1277         d.addCallback(_check)
1278         return d
1279
1280     def test_PUT_NEWDIRURL(self):
1281         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1282         d.addCallback(lambda res:
1283                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1284         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1285         d.addCallback(self.failUnlessNodeKeysAre, [])
1286         return d
1287
1288     def test_POST_NEWDIRURL(self):
1289         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1290         d.addCallback(lambda res:
1291                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1292         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1293         d.addCallback(self.failUnlessNodeKeysAre, [])
1294         return d
1295
1296     def test_POST_NEWDIRURL_emptyname(self):
1297         # an empty pathname component (i.e. a double-slash) is disallowed
1298         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1299                              "400 Bad Request",
1300                              "The webapi does not allow empty pathname components, i.e. a double slash",
1301                              self.POST, self.public_url + "//?t=mkdir")
1302         return d
1303
1304     def test_POST_NEWDIRURL_initial_children(self):
1305         (newkids, caps) = self._create_initial_children()
1306         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1307                        simplejson.dumps(newkids))
1308         def _check(uri):
1309             n = self.s.create_node_from_uri(uri.strip())
1310             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1311             d2.addCallback(lambda ign:
1312                            self.failUnlessROChildURIIs(n, u"child-imm",
1313                                                        caps['filecap1']))
1314             d2.addCallback(lambda ign:
1315                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1316                                                        caps['filecap2']))
1317             d2.addCallback(lambda ign:
1318                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1319                                                        caps['filecap3']))
1320             d2.addCallback(lambda ign:
1321                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1322                                                        caps['unknown_rocap']))
1323             d2.addCallback(lambda ign:
1324                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1325                                                        caps['unknown_rwcap']))
1326             d2.addCallback(lambda ign:
1327                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1328                                                        caps['unknown_immcap']))
1329             d2.addCallback(lambda ign:
1330                            self.failUnlessRWChildURIIs(n, u"dirchild",
1331                                                        caps['dircap']))
1332             d2.addCallback(lambda ign:
1333                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1334                                                        caps['litdircap']))
1335             d2.addCallback(lambda ign:
1336                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1337                                                        caps['emptydircap']))
1338             return d2
1339         d.addCallback(_check)
1340         d.addCallback(lambda res:
1341                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1342         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1343         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1344         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1345         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1346         return d
1347
1348     def test_POST_NEWDIRURL_immutable(self):
1349         (newkids, caps) = self._create_immutable_children()
1350         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1351                        simplejson.dumps(newkids))
1352         def _check(uri):
1353             n = self.s.create_node_from_uri(uri.strip())
1354             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1355             d2.addCallback(lambda ign:
1356                            self.failUnlessROChildURIIs(n, u"child-imm",
1357                                                        caps['filecap1']))
1358             d2.addCallback(lambda ign:
1359                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1360                                                        caps['unknown_immcap']))
1361             d2.addCallback(lambda ign:
1362                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1363                                                        caps['immdircap']))
1364             d2.addCallback(lambda ign:
1365                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1366                                                        caps['litdircap']))
1367             d2.addCallback(lambda ign:
1368                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1369                                                        caps['emptydircap']))
1370             return d2
1371         d.addCallback(_check)
1372         d.addCallback(lambda res:
1373                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1374         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1375         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1376         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1377         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1378         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1379         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1380         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1381         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1382         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1383         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1384         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1385         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1386         d.addErrback(self.explain_web_error)
1387         return d
1388
1389     def test_POST_NEWDIRURL_immutable_bad(self):
1390         (newkids, caps) = self._create_initial_children()
1391         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1392                              "400 Bad Request",
1393                              "needed to be immutable but was not",
1394                              self.POST2,
1395                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1396                              simplejson.dumps(newkids))
1397         return d
1398
1399     def test_PUT_NEWDIRURL_exists(self):
1400         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1401         d.addCallback(lambda res:
1402                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1403         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1404         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1405         return d
1406
1407     def test_PUT_NEWDIRURL_blocked(self):
1408         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1409                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1410                              self.PUT,
1411                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1412         d.addCallback(lambda res:
1413                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1414         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1415         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1416         return d
1417
1418     def test_PUT_NEWDIRURL_mkdir_p(self):
1419         d = defer.succeed(None)
1420         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1421         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1422         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1423         def mkdir_p(mkpnode):
1424             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1425             d = self.POST(url)
1426             def made_subsub(ssuri):
1427                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1428                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1429                 d = self.POST(url)
1430                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1431                 return d
1432             d.addCallback(made_subsub)
1433             return d
1434         d.addCallback(mkdir_p)
1435         return d
1436
1437     def test_PUT_NEWDIRURL_mkdirs(self):
1438         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1439         d.addCallback(lambda res:
1440                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1441         d.addCallback(lambda res:
1442                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1443         d.addCallback(lambda res:
1444                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1445         d.addCallback(self.failUnlessNodeKeysAre, [])
1446         return d
1447
1448     def test_DELETE_DIRURL(self):
1449         d = self.DELETE(self.public_url + "/foo")
1450         d.addCallback(lambda res:
1451                       self.failIfNodeHasChild(self.public_root, u"foo"))
1452         return d
1453
1454     def test_DELETE_DIRURL_missing(self):
1455         d = self.DELETE(self.public_url + "/foo/missing")
1456         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1457         d.addCallback(lambda res:
1458                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1459         return d
1460
1461     def test_DELETE_DIRURL_missing2(self):
1462         d = self.DELETE(self.public_url + "/missing")
1463         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1464         return d
1465
1466     def dump_root(self):
1467         print "NODEWALK"
1468         w = webish.DirnodeWalkerMixin()
1469         def visitor(childpath, childnode, metadata):
1470             print childpath
1471         d = w.walk(self.public_root, visitor)
1472         return d
1473
1474     def failUnlessNodeKeysAre(self, node, expected_keys):
1475         for k in expected_keys:
1476             assert isinstance(k, unicode)
1477         d = node.list()
1478         def _check(children):
1479             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1480         d.addCallback(_check)
1481         return d
1482     def failUnlessNodeHasChild(self, node, name):
1483         assert isinstance(name, unicode)
1484         d = node.list()
1485         def _check(children):
1486             self.failUnless(name in children)
1487         d.addCallback(_check)
1488         return d
1489     def failIfNodeHasChild(self, node, name):
1490         assert isinstance(name, unicode)
1491         d = node.list()
1492         def _check(children):
1493             self.failIf(name in children)
1494         d.addCallback(_check)
1495         return d
1496
1497     def failUnlessChildContentsAre(self, node, name, expected_contents):
1498         assert isinstance(name, unicode)
1499         d = node.get_child_at_path(name)
1500         d.addCallback(lambda node: download_to_data(node))
1501         def _check(contents):
1502             self.failUnlessReallyEqual(contents, expected_contents)
1503         d.addCallback(_check)
1504         return d
1505
1506     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1507         assert isinstance(name, unicode)
1508         d = node.get_child_at_path(name)
1509         d.addCallback(lambda node: node.download_best_version())
1510         def _check(contents):
1511             self.failUnlessReallyEqual(contents, expected_contents)
1512         d.addCallback(_check)
1513         return d
1514
1515     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1516         assert isinstance(name, unicode)
1517         d = node.get_child_at_path(name)
1518         def _check(child):
1519             self.failUnless(child.is_unknown() or not child.is_readonly())
1520             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1521             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1522             expected_ro_uri = self._make_readonly(expected_uri)
1523             if expected_ro_uri:
1524                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1525         d.addCallback(_check)
1526         return d
1527
1528     def failUnlessROChildURIIs(self, node, name, expected_uri):
1529         assert isinstance(name, unicode)
1530         d = node.get_child_at_path(name)
1531         def _check(child):
1532             self.failUnless(child.is_unknown() or child.is_readonly())
1533             self.failUnlessReallyEqual(child.get_write_uri(), None)
1534             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1535             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1536         d.addCallback(_check)
1537         return d
1538
1539     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1540         assert isinstance(name, unicode)
1541         d = node.get_child_at_path(name)
1542         def _check(child):
1543             self.failUnless(child.is_unknown() or not child.is_readonly())
1544             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1545             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1546             expected_ro_uri = self._make_readonly(got_uri)
1547             if expected_ro_uri:
1548                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1549         d.addCallback(_check)
1550         return d
1551
1552     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1553         assert isinstance(name, unicode)
1554         d = node.get_child_at_path(name)
1555         def _check(child):
1556             self.failUnless(child.is_unknown() or child.is_readonly())
1557             self.failUnlessReallyEqual(child.get_write_uri(), None)
1558             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1559             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1560         d.addCallback(_check)
1561         return d
1562
1563     def failUnlessCHKURIHasContents(self, got_uri, contents):
1564         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1565
1566     def test_POST_upload(self):
1567         d = self.POST(self.public_url + "/foo", t="upload",
1568                       file=("new.txt", self.NEWFILE_CONTENTS))
1569         fn = self._foo_node
1570         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1571         d.addCallback(lambda res:
1572                       self.failUnlessChildContentsAre(fn, u"new.txt",
1573                                                       self.NEWFILE_CONTENTS))
1574         return d
1575
1576     def test_POST_upload_unicode(self):
1577         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1578         d = self.POST(self.public_url + "/foo", t="upload",
1579                       file=(filename, self.NEWFILE_CONTENTS))
1580         fn = self._foo_node
1581         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1582         d.addCallback(lambda res:
1583                       self.failUnlessChildContentsAre(fn, filename,
1584                                                       self.NEWFILE_CONTENTS))
1585         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1586         d.addCallback(lambda res: self.GET(target_url))
1587         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1588                                                                   self.NEWFILE_CONTENTS,
1589                                                                   contents))
1590         return d
1591
1592     def test_POST_upload_unicode_named(self):
1593         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1594         d = self.POST(self.public_url + "/foo", t="upload",
1595                       name=filename,
1596                       file=("overridden", self.NEWFILE_CONTENTS))
1597         fn = self._foo_node
1598         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1599         d.addCallback(lambda res:
1600                       self.failUnlessChildContentsAre(fn, filename,
1601                                                       self.NEWFILE_CONTENTS))
1602         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1603         d.addCallback(lambda res: self.GET(target_url))
1604         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1605                                                                   self.NEWFILE_CONTENTS,
1606                                                                   contents))
1607         return d
1608
1609     def test_POST_upload_no_link(self):
1610         d = self.POST("/uri", t="upload",
1611                       file=("new.txt", self.NEWFILE_CONTENTS))
1612         def _check_upload_results(page):
1613             # this should be a page which describes the results of the upload
1614             # that just finished.
1615             self.failUnless("Upload Results:" in page)
1616             self.failUnless("URI:" in page)
1617             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1618             mo = uri_re.search(page)
1619             self.failUnless(mo, page)
1620             new_uri = mo.group(1)
1621             return new_uri
1622         d.addCallback(_check_upload_results)
1623         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1624         return d
1625
1626     def test_POST_upload_no_link_whendone(self):
1627         d = self.POST("/uri", t="upload", when_done="/",
1628                       file=("new.txt", self.NEWFILE_CONTENTS))
1629         d.addBoth(self.shouldRedirect, "/")
1630         return d
1631
1632     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1633         d = defer.maybeDeferred(callable, *args, **kwargs)
1634         def done(res):
1635             if isinstance(res, failure.Failure):
1636                 res.trap(error.PageRedirect)
1637                 statuscode = res.value.status
1638                 target = res.value.location
1639                 return checker(statuscode, target)
1640             self.fail("%s: callable was supposed to redirect, not return '%s'"
1641                       % (which, res))
1642         d.addBoth(done)
1643         return d
1644
1645     def test_POST_upload_no_link_whendone_results(self):
1646         def check(statuscode, target):
1647             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1648             self.failUnless(target.startswith(self.webish_url), target)
1649             return client.getPage(target, method="GET")
1650         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1651                                  check,
1652                                  self.POST, "/uri", t="upload",
1653                                  when_done="/uri/%(uri)s",
1654                                  file=("new.txt", self.NEWFILE_CONTENTS))
1655         d.addCallback(lambda res:
1656                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1657         return d
1658
1659     def test_POST_upload_no_link_mutable(self):
1660         d = self.POST("/uri", t="upload", mutable="true",
1661                       file=("new.txt", self.NEWFILE_CONTENTS))
1662         def _check(filecap):
1663             filecap = filecap.strip()
1664             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1665             self.filecap = filecap
1666             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1667             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1668             n = self.s.create_node_from_uri(filecap)
1669             return n.download_best_version()
1670         d.addCallback(_check)
1671         def _check2(data):
1672             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1673             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1674         d.addCallback(_check2)
1675         def _check3(data):
1676             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1677             return self.GET("/file/%s" % urllib.quote(self.filecap))
1678         d.addCallback(_check3)
1679         def _check4(data):
1680             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1681         d.addCallback(_check4)
1682         return d
1683
1684     def test_POST_upload_no_link_mutable_toobig(self):
1685         d = self.shouldFail2(error.Error,
1686                              "test_POST_upload_no_link_mutable_toobig",
1687                              "413 Request Entity Too Large",
1688                              "SDMF is limited to one segment, and 10001 > 10000",
1689                              self.POST,
1690                              "/uri", t="upload", mutable="true",
1691                              file=("new.txt",
1692                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1693         return d
1694
1695     def test_POST_upload_mutable(self):
1696         # this creates a mutable file
1697         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1698                       file=("new.txt", self.NEWFILE_CONTENTS))
1699         fn = self._foo_node
1700         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1701         d.addCallback(lambda res:
1702                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1703                                                              self.NEWFILE_CONTENTS))
1704         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1705         def _got(newnode):
1706             self.failUnless(IMutableFileNode.providedBy(newnode))
1707             self.failUnless(newnode.is_mutable())
1708             self.failIf(newnode.is_readonly())
1709             self._mutable_node = newnode
1710             self._mutable_uri = newnode.get_uri()
1711         d.addCallback(_got)
1712
1713         # now upload it again and make sure that the URI doesn't change
1714         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1715         d.addCallback(lambda res:
1716                       self.POST(self.public_url + "/foo", t="upload",
1717                                 mutable="true",
1718                                 file=("new.txt", NEWER_CONTENTS)))
1719         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1720         d.addCallback(lambda res:
1721                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1722                                                              NEWER_CONTENTS))
1723         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1724         def _got2(newnode):
1725             self.failUnless(IMutableFileNode.providedBy(newnode))
1726             self.failUnless(newnode.is_mutable())
1727             self.failIf(newnode.is_readonly())
1728             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1729         d.addCallback(_got2)
1730
1731         # upload a second time, using PUT instead of POST
1732         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1733         d.addCallback(lambda res:
1734                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1735         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1736         d.addCallback(lambda res:
1737                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1738                                                              NEW2_CONTENTS))
1739
1740         # finally list the directory, since mutable files are displayed
1741         # slightly differently
1742
1743         d.addCallback(lambda res:
1744                       self.GET(self.public_url + "/foo/",
1745                                followRedirect=True))
1746         def _check_page(res):
1747             # TODO: assert more about the contents
1748             self.failUnless("SSK" in res)
1749             return res
1750         d.addCallback(_check_page)
1751
1752         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1753         def _got3(newnode):
1754             self.failUnless(IMutableFileNode.providedBy(newnode))
1755             self.failUnless(newnode.is_mutable())
1756             self.failIf(newnode.is_readonly())
1757             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1758         d.addCallback(_got3)
1759
1760         # look at the JSON form of the enclosing directory
1761         d.addCallback(lambda res:
1762                       self.GET(self.public_url + "/foo/?t=json",
1763                                followRedirect=True))
1764         def _check_page_json(res):
1765             parsed = simplejson.loads(res)
1766             self.failUnlessEqual(parsed[0], "dirnode")
1767             children = dict( [(unicode(name),value)
1768                               for (name,value)
1769                               in parsed[1]["children"].iteritems()] )
1770             self.failUnless(u"new.txt" in children)
1771             new_json = children[u"new.txt"]
1772             self.failUnlessEqual(new_json[0], "filenode")
1773             self.failUnless(new_json[1]["mutable"])
1774             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
1775             ro_uri = self._mutable_node.get_readonly().to_string()
1776             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
1777         d.addCallback(_check_page_json)
1778
1779         # and the JSON form of the file
1780         d.addCallback(lambda res:
1781                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1782         def _check_file_json(res):
1783             parsed = simplejson.loads(res)
1784             self.failUnlessEqual(parsed[0], "filenode")
1785             self.failUnless(parsed[1]["mutable"])
1786             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
1787             ro_uri = self._mutable_node.get_readonly().to_string()
1788             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
1789         d.addCallback(_check_file_json)
1790
1791         # and look at t=uri and t=readonly-uri
1792         d.addCallback(lambda res:
1793                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1794         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1795         d.addCallback(lambda res:
1796                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1797         def _check_ro_uri(res):
1798             ro_uri = self._mutable_node.get_readonly().to_string()
1799             self.failUnlessReallyEqual(res, ro_uri)
1800         d.addCallback(_check_ro_uri)
1801
1802         # make sure we can get to it from /uri/URI
1803         d.addCallback(lambda res:
1804                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1805         d.addCallback(lambda res:
1806                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1807
1808         # and that HEAD computes the size correctly
1809         d.addCallback(lambda res:
1810                       self.HEAD(self.public_url + "/foo/new.txt",
1811                                 return_response=True))
1812         def _got_headers((res, status, headers)):
1813             self.failUnlessReallyEqual(res, "")
1814             self.failUnlessReallyEqual(headers["content-length"][0],
1815                                        str(len(NEW2_CONTENTS)))
1816             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1817         d.addCallback(_got_headers)
1818
1819         # make sure that size errors are displayed correctly for overwrite
1820         d.addCallback(lambda res:
1821                       self.shouldFail2(error.Error,
1822                                        "test_POST_upload_mutable-toobig",
1823                                        "413 Request Entity Too Large",
1824                                        "SDMF is limited to one segment, and 10001 > 10000",
1825                                        self.POST,
1826                                        self.public_url + "/foo", t="upload",
1827                                        mutable="true",
1828                                        file=("new.txt",
1829                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1830                                        ))
1831
1832         d.addErrback(self.dump_error)
1833         return d
1834
1835     def test_POST_upload_mutable_toobig(self):
1836         d = self.shouldFail2(error.Error,
1837                              "test_POST_upload_mutable_toobig",
1838                              "413 Request Entity Too Large",
1839                              "SDMF is limited to one segment, and 10001 > 10000",
1840                              self.POST,
1841                              self.public_url + "/foo",
1842                              t="upload", mutable="true",
1843                              file=("new.txt",
1844                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1845         return d
1846
1847     def dump_error(self, f):
1848         # if the web server returns an error code (like 400 Bad Request),
1849         # web.client.getPage puts the HTTP response body into the .response
1850         # attribute of the exception object that it gives back. It does not
1851         # appear in the Failure's repr(), so the ERROR that trial displays
1852         # will be rather terse and unhelpful. addErrback this method to the
1853         # end of your chain to get more information out of these errors.
1854         if f.check(error.Error):
1855             print "web.error.Error:"
1856             print f
1857             print f.value.response
1858         return f
1859
1860     def test_POST_upload_replace(self):
1861         d = self.POST(self.public_url + "/foo", t="upload",
1862                       file=("bar.txt", self.NEWFILE_CONTENTS))
1863         fn = self._foo_node
1864         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1865         d.addCallback(lambda res:
1866                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1867                                                       self.NEWFILE_CONTENTS))
1868         return d
1869
1870     def test_POST_upload_no_replace_ok(self):
1871         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1872                       file=("new.txt", self.NEWFILE_CONTENTS))
1873         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1874         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1875                                                              self.NEWFILE_CONTENTS))
1876         return d
1877
1878     def test_POST_upload_no_replace_queryarg(self):
1879         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1880                       file=("bar.txt", self.NEWFILE_CONTENTS))
1881         d.addBoth(self.shouldFail, error.Error,
1882                   "POST_upload_no_replace_queryarg",
1883                   "409 Conflict",
1884                   "There was already a child by that name, and you asked me "
1885                   "to not replace it")
1886         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1887         d.addCallback(self.failUnlessIsBarDotTxt)
1888         return d
1889
1890     def test_POST_upload_no_replace_field(self):
1891         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1892                       file=("bar.txt", self.NEWFILE_CONTENTS))
1893         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1894                   "409 Conflict",
1895                   "There was already a child by that name, and you asked me "
1896                   "to not replace it")
1897         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1898         d.addCallback(self.failUnlessIsBarDotTxt)
1899         return d
1900
1901     def test_POST_upload_whendone(self):
1902         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1903                       file=("new.txt", self.NEWFILE_CONTENTS))
1904         d.addBoth(self.shouldRedirect, "/THERE")
1905         fn = self._foo_node
1906         d.addCallback(lambda res:
1907                       self.failUnlessChildContentsAre(fn, u"new.txt",
1908                                                       self.NEWFILE_CONTENTS))
1909         return d
1910
1911     def test_POST_upload_named(self):
1912         fn = self._foo_node
1913         d = self.POST(self.public_url + "/foo", t="upload",
1914                       name="new.txt", file=self.NEWFILE_CONTENTS)
1915         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1916         d.addCallback(lambda res:
1917                       self.failUnlessChildContentsAre(fn, u"new.txt",
1918                                                       self.NEWFILE_CONTENTS))
1919         return d
1920
1921     def test_POST_upload_named_badfilename(self):
1922         d = self.POST(self.public_url + "/foo", t="upload",
1923                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1924         d.addBoth(self.shouldFail, error.Error,
1925                   "test_POST_upload_named_badfilename",
1926                   "400 Bad Request",
1927                   "name= may not contain a slash",
1928                   )
1929         # make sure that nothing was added
1930         d.addCallback(lambda res:
1931                       self.failUnlessNodeKeysAre(self._foo_node,
1932                                                  [u"bar.txt", u"blockingfile",
1933                                                   u"empty", u"n\u00fc.txt",
1934                                                   u"sub"]))
1935         return d
1936
1937     def test_POST_FILEURL_check(self):
1938         bar_url = self.public_url + "/foo/bar.txt"
1939         d = self.POST(bar_url, t="check")
1940         def _check(res):
1941             self.failUnless("Healthy :" in res)
1942         d.addCallback(_check)
1943         redir_url = "http://allmydata.org/TARGET"
1944         def _check2(statuscode, target):
1945             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1946             self.failUnlessReallyEqual(target, redir_url)
1947         d.addCallback(lambda res:
1948                       self.shouldRedirect2("test_POST_FILEURL_check",
1949                                            _check2,
1950                                            self.POST, bar_url,
1951                                            t="check",
1952                                            when_done=redir_url))
1953         d.addCallback(lambda res:
1954                       self.POST(bar_url, t="check", return_to=redir_url))
1955         def _check3(res):
1956             self.failUnless("Healthy :" in res)
1957             self.failUnless("Return to file" in res)
1958             self.failUnless(redir_url in res)
1959         d.addCallback(_check3)
1960
1961         d.addCallback(lambda res:
1962                       self.POST(bar_url, t="check", output="JSON"))
1963         def _check_json(res):
1964             data = simplejson.loads(res)
1965             self.failUnless("storage-index" in data)
1966             self.failUnless(data["results"]["healthy"])
1967         d.addCallback(_check_json)
1968
1969         return d
1970
1971     def test_POST_FILEURL_check_and_repair(self):
1972         bar_url = self.public_url + "/foo/bar.txt"
1973         d = self.POST(bar_url, t="check", repair="true")
1974         def _check(res):
1975             self.failUnless("Healthy :" in res)
1976         d.addCallback(_check)
1977         redir_url = "http://allmydata.org/TARGET"
1978         def _check2(statuscode, target):
1979             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1980             self.failUnlessReallyEqual(target, redir_url)
1981         d.addCallback(lambda res:
1982                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1983                                            _check2,
1984                                            self.POST, bar_url,
1985                                            t="check", repair="true",
1986                                            when_done=redir_url))
1987         d.addCallback(lambda res:
1988                       self.POST(bar_url, t="check", return_to=redir_url))
1989         def _check3(res):
1990             self.failUnless("Healthy :" in res)
1991             self.failUnless("Return to file" in res)
1992             self.failUnless(redir_url in res)
1993         d.addCallback(_check3)
1994         return d
1995
1996     def test_POST_DIRURL_check(self):
1997         foo_url = self.public_url + "/foo/"
1998         d = self.POST(foo_url, t="check")
1999         def _check(res):
2000             self.failUnless("Healthy :" in res, res)
2001         d.addCallback(_check)
2002         redir_url = "http://allmydata.org/TARGET"
2003         def _check2(statuscode, target):
2004             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2005             self.failUnlessReallyEqual(target, redir_url)
2006         d.addCallback(lambda res:
2007                       self.shouldRedirect2("test_POST_DIRURL_check",
2008                                            _check2,
2009                                            self.POST, foo_url,
2010                                            t="check",
2011                                            when_done=redir_url))
2012         d.addCallback(lambda res:
2013                       self.POST(foo_url, t="check", return_to=redir_url))
2014         def _check3(res):
2015             self.failUnless("Healthy :" in res, res)
2016             self.failUnless("Return to file/directory" in res)
2017             self.failUnless(redir_url in res)
2018         d.addCallback(_check3)
2019
2020         d.addCallback(lambda res:
2021                       self.POST(foo_url, t="check", output="JSON"))
2022         def _check_json(res):
2023             data = simplejson.loads(res)
2024             self.failUnless("storage-index" in data)
2025             self.failUnless(data["results"]["healthy"])
2026         d.addCallback(_check_json)
2027
2028         return d
2029
2030     def test_POST_DIRURL_check_and_repair(self):
2031         foo_url = self.public_url + "/foo/"
2032         d = self.POST(foo_url, t="check", repair="true")
2033         def _check(res):
2034             self.failUnless("Healthy :" in res, res)
2035         d.addCallback(_check)
2036         redir_url = "http://allmydata.org/TARGET"
2037         def _check2(statuscode, target):
2038             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2039             self.failUnlessReallyEqual(target, redir_url)
2040         d.addCallback(lambda res:
2041                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2042                                            _check2,
2043                                            self.POST, foo_url,
2044                                            t="check", repair="true",
2045                                            when_done=redir_url))
2046         d.addCallback(lambda res:
2047                       self.POST(foo_url, t="check", return_to=redir_url))
2048         def _check3(res):
2049             self.failUnless("Healthy :" in res)
2050             self.failUnless("Return to file/directory" in res)
2051             self.failUnless(redir_url in res)
2052         d.addCallback(_check3)
2053         return d
2054
2055     def wait_for_operation(self, ignored, ophandle):
2056         url = "/operations/" + ophandle
2057         url += "?t=status&output=JSON"
2058         d = self.GET(url)
2059         def _got(res):
2060             data = simplejson.loads(res)
2061             if not data["finished"]:
2062                 d = self.stall(delay=1.0)
2063                 d.addCallback(self.wait_for_operation, ophandle)
2064                 return d
2065             return data
2066         d.addCallback(_got)
2067         return d
2068
2069     def get_operation_results(self, ignored, ophandle, output=None):
2070         url = "/operations/" + ophandle
2071         url += "?t=status"
2072         if output:
2073             url += "&output=" + output
2074         d = self.GET(url)
2075         def _got(res):
2076             if output and output.lower() == "json":
2077                 return simplejson.loads(res)
2078             return res
2079         d.addCallback(_got)
2080         return d
2081
2082     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2083         d = self.shouldFail2(error.Error,
2084                              "test_POST_DIRURL_deepcheck_no_ophandle",
2085                              "400 Bad Request",
2086                              "slow operation requires ophandle=",
2087                              self.POST, self.public_url, t="start-deep-check")
2088         return d
2089
2090     def test_POST_DIRURL_deepcheck(self):
2091         def _check_redirect(statuscode, target):
2092             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2093             self.failUnless(target.endswith("/operations/123"))
2094         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2095                                  self.POST, self.public_url,
2096                                  t="start-deep-check", ophandle="123")
2097         d.addCallback(self.wait_for_operation, "123")
2098         def _check_json(data):
2099             self.failUnlessReallyEqual(data["finished"], True)
2100             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2101             self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2102         d.addCallback(_check_json)
2103         d.addCallback(self.get_operation_results, "123", "html")
2104         def _check_html(res):
2105             self.failUnless("Objects Checked: <span>8</span>" in res)
2106             self.failUnless("Objects Healthy: <span>8</span>" in res)
2107         d.addCallback(_check_html)
2108
2109         d.addCallback(lambda res:
2110                       self.GET("/operations/123/"))
2111         d.addCallback(_check_html) # should be the same as without the slash
2112
2113         d.addCallback(lambda res:
2114                       self.shouldFail2(error.Error, "one", "404 Not Found",
2115                                        "No detailed results for SI bogus",
2116                                        self.GET, "/operations/123/bogus"))
2117
2118         foo_si = self._foo_node.get_storage_index()
2119         foo_si_s = base32.b2a(foo_si)
2120         d.addCallback(lambda res:
2121                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2122         def _check_foo_json(res):
2123             data = simplejson.loads(res)
2124             self.failUnlessEqual(data["storage-index"], foo_si_s)
2125             self.failUnless(data["results"]["healthy"])
2126         d.addCallback(_check_foo_json)
2127         return d
2128
2129     def test_POST_DIRURL_deepcheck_and_repair(self):
2130         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2131                       ophandle="124", output="json", followRedirect=True)
2132         d.addCallback(self.wait_for_operation, "124")
2133         def _check_json(data):
2134             self.failUnlessReallyEqual(data["finished"], True)
2135             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2136             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2137             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2138             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2139             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2140             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2141             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2142             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2143             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2144             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2145         d.addCallback(_check_json)
2146         d.addCallback(self.get_operation_results, "124", "html")
2147         def _check_html(res):
2148             self.failUnless("Objects Checked: <span>8</span>" in res)
2149
2150             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2151             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2152             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2153
2154             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2155             self.failUnless("Repairs Successful: <span>0</span>" in res)
2156             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2157
2158             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2159             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2160             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2161         d.addCallback(_check_html)
2162         return d
2163
2164     def test_POST_FILEURL_bad_t(self):
2165         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2166                              "POST to file: bad t=bogus",
2167                              self.POST, self.public_url + "/foo/bar.txt",
2168                              t="bogus")
2169         return d
2170
2171     def test_POST_mkdir(self): # return value?
2172         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2173         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2174         d.addCallback(self.failUnlessNodeKeysAre, [])
2175         return d
2176
2177     def test_POST_mkdir_initial_children(self):
2178         (newkids, caps) = self._create_initial_children()
2179         d = self.POST2(self.public_url +
2180                        "/foo?t=mkdir-with-children&name=newdir",
2181                        simplejson.dumps(newkids))
2182         d.addCallback(lambda res:
2183                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2184         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2185         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2186         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2187         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2188         return d
2189
2190     def test_POST_mkdir_immutable(self):
2191         (newkids, caps) = self._create_immutable_children()
2192         d = self.POST2(self.public_url +
2193                        "/foo?t=mkdir-immutable&name=newdir",
2194                        simplejson.dumps(newkids))
2195         d.addCallback(lambda res:
2196                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2197         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2198         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2199         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2200         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2201         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2202         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2203         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2204         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2205         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2206         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2207         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2208         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2209         return d
2210
2211     def test_POST_mkdir_immutable_bad(self):
2212         (newkids, caps) = self._create_initial_children()
2213         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2214                              "400 Bad Request",
2215                              "needed to be immutable but was not",
2216                              self.POST2,
2217                              self.public_url +
2218                              "/foo?t=mkdir-immutable&name=newdir",
2219                              simplejson.dumps(newkids))
2220         return d
2221
2222     def test_POST_mkdir_2(self):
2223         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2224         d.addCallback(lambda res:
2225                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2226         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2227         d.addCallback(self.failUnlessNodeKeysAre, [])
2228         return d
2229
2230     def test_POST_mkdirs_2(self):
2231         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2232         d.addCallback(lambda res:
2233                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2234         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2235         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2236         d.addCallback(self.failUnlessNodeKeysAre, [])
2237         return d
2238
2239     def test_POST_mkdir_no_parentdir_noredirect(self):
2240         d = self.POST("/uri?t=mkdir")
2241         def _after_mkdir(res):
2242             uri.DirectoryURI.init_from_string(res)
2243         d.addCallback(_after_mkdir)
2244         return d
2245
2246     def test_POST_mkdir_no_parentdir_noredirect2(self):
2247         # make sure form-based arguments (as on the welcome page) still work
2248         d = self.POST("/uri", t="mkdir")
2249         def _after_mkdir(res):
2250             uri.DirectoryURI.init_from_string(res)
2251         d.addCallback(_after_mkdir)
2252         d.addErrback(self.explain_web_error)
2253         return d
2254
2255     def test_POST_mkdir_no_parentdir_redirect(self):
2256         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2257         d.addBoth(self.shouldRedirect, None, statuscode='303')
2258         def _check_target(target):
2259             target = urllib.unquote(target)
2260             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2261         d.addCallback(_check_target)
2262         return d
2263
2264     def test_POST_mkdir_no_parentdir_redirect2(self):
2265         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2266         d.addBoth(self.shouldRedirect, None, statuscode='303')
2267         def _check_target(target):
2268             target = urllib.unquote(target)
2269             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2270         d.addCallback(_check_target)
2271         d.addErrback(self.explain_web_error)
2272         return d
2273
2274     def _make_readonly(self, u):
2275         ro_uri = uri.from_string(u).get_readonly()
2276         if ro_uri is None:
2277             return None
2278         return ro_uri.to_string()
2279
2280     def _create_initial_children(self):
2281         contents, n, filecap1 = self.makefile(12)
2282         md1 = {"metakey1": "metavalue1"}
2283         filecap2 = make_mutable_file_uri()
2284         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2285         filecap3 = node3.get_readonly_uri()
2286         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2287         dircap = DirectoryNode(node4, None, None).get_uri()
2288         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2289         emptydircap = "URI:DIR2-LIT:"
2290         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2291                                                       "ro_uri": self._make_readonly(filecap1),
2292                                                       "metadata": md1, }],
2293                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2294                                                       "ro_uri": self._make_readonly(filecap2)}],
2295                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2296                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2297                                                       "ro_uri": unknown_rocap}],
2298                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2299                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2300                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2301                                                       "ro_uri": self._make_readonly(dircap)}],
2302                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2303                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2304                    }
2305         return newkids, {'filecap1': filecap1,
2306                          'filecap2': filecap2,
2307                          'filecap3': filecap3,
2308                          'unknown_rwcap': unknown_rwcap,
2309                          'unknown_rocap': unknown_rocap,
2310                          'unknown_immcap': unknown_immcap,
2311                          'dircap': dircap,
2312                          'litdircap': litdircap,
2313                          'emptydircap': emptydircap}
2314
2315     def _create_immutable_children(self):
2316         contents, n, filecap1 = self.makefile(12)
2317         md1 = {"metakey1": "metavalue1"}
2318         tnode = create_chk_filenode("immutable directory contents\n"*10)
2319         dnode = DirectoryNode(tnode, None, None)
2320         assert not dnode.is_mutable()
2321         immdircap = dnode.get_uri()
2322         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2323         emptydircap = "URI:DIR2-LIT:"
2324         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2325                                                       "metadata": md1, }],
2326                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2327                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2328                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2329                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2330                    }
2331         return newkids, {'filecap1': filecap1,
2332                          'unknown_immcap': unknown_immcap,
2333                          'immdircap': immdircap,
2334                          'litdircap': litdircap,
2335                          'emptydircap': emptydircap}
2336
2337     def test_POST_mkdir_no_parentdir_initial_children(self):
2338         (newkids, caps) = self._create_initial_children()
2339         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2340         def _after_mkdir(res):
2341             self.failUnless(res.startswith("URI:DIR"), res)
2342             n = self.s.create_node_from_uri(res)
2343             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2344             d2.addCallback(lambda ign:
2345                            self.failUnlessROChildURIIs(n, u"child-imm",
2346                                                        caps['filecap1']))
2347             d2.addCallback(lambda ign:
2348                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2349                                                        caps['filecap2']))
2350             d2.addCallback(lambda ign:
2351                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2352                                                        caps['filecap3']))
2353             d2.addCallback(lambda ign:
2354                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2355                                                        caps['unknown_rwcap']))
2356             d2.addCallback(lambda ign:
2357                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2358                                                        caps['unknown_rocap']))
2359             d2.addCallback(lambda ign:
2360                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2361                                                        caps['unknown_immcap']))
2362             d2.addCallback(lambda ign:
2363                            self.failUnlessRWChildURIIs(n, u"dirchild",
2364                                                        caps['dircap']))
2365             return d2
2366         d.addCallback(_after_mkdir)
2367         return d
2368
2369     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2370         # the regular /uri?t=mkdir operation is specified to ignore its body.
2371         # Only t=mkdir-with-children pays attention to it.
2372         (newkids, caps) = self._create_initial_children()
2373         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2374                                  400, "Bad Request",
2375                                  "t=mkdir does not accept children=, "
2376                                  "try t=mkdir-with-children instead",
2377                                  self.POST2, "/uri?t=mkdir", # without children
2378                                  simplejson.dumps(newkids))
2379         return d
2380
2381     def test_POST_noparent_bad(self):
2382         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2383                                  "/uri accepts only PUT, PUT?t=mkdir, "
2384                                  "POST?t=upload, and POST?t=mkdir",
2385                                  self.POST, "/uri?t=bogus")
2386         return d
2387
2388     def test_POST_mkdir_no_parentdir_immutable(self):
2389         (newkids, caps) = self._create_immutable_children()
2390         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2391         def _after_mkdir(res):
2392             self.failUnless(res.startswith("URI:DIR"), res)
2393             n = self.s.create_node_from_uri(res)
2394             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2395             d2.addCallback(lambda ign:
2396                            self.failUnlessROChildURIIs(n, u"child-imm",
2397                                                           caps['filecap1']))
2398             d2.addCallback(lambda ign:
2399                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2400                                                           caps['unknown_immcap']))
2401             d2.addCallback(lambda ign:
2402                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2403                                                           caps['immdircap']))
2404             d2.addCallback(lambda ign:
2405                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2406                                                           caps['litdircap']))
2407             d2.addCallback(lambda ign:
2408                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2409                                                           caps['emptydircap']))
2410             return d2
2411         d.addCallback(_after_mkdir)
2412         return d
2413
2414     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2415         (newkids, caps) = self._create_initial_children()
2416         d = self.shouldFail2(error.Error,
2417                              "test_POST_mkdir_no_parentdir_immutable_bad",
2418                              "400 Bad Request",
2419                              "needed to be immutable but was not",
2420                              self.POST2,
2421                              "/uri?t=mkdir-immutable",
2422                              simplejson.dumps(newkids))
2423         return d
2424
2425     def test_welcome_page_mkdir_button(self):
2426         # Fetch the welcome page.
2427         d = self.GET("/")
2428         def _after_get_welcome_page(res):
2429             MKDIR_BUTTON_RE = re.compile(
2430                 '<form action="([^"]*)" method="post".*?'
2431                 '<input type="hidden" name="t" value="([^"]*)" />'
2432                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2433                 '<input type="submit" value="Create a directory" />',
2434                 re.I)
2435             mo = MKDIR_BUTTON_RE.search(res)
2436             formaction = mo.group(1)
2437             formt = mo.group(2)
2438             formaname = mo.group(3)
2439             formavalue = mo.group(4)
2440             return (formaction, formt, formaname, formavalue)
2441         d.addCallback(_after_get_welcome_page)
2442         def _after_parse_form(res):
2443             (formaction, formt, formaname, formavalue) = res
2444             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2445         d.addCallback(_after_parse_form)
2446         d.addBoth(self.shouldRedirect, None, statuscode='303')
2447         return d
2448
2449     def test_POST_mkdir_replace(self): # return value?
2450         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2451         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2452         d.addCallback(self.failUnlessNodeKeysAre, [])
2453         return d
2454
2455     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2456         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2457         d.addBoth(self.shouldFail, error.Error,
2458                   "POST_mkdir_no_replace_queryarg",
2459                   "409 Conflict",
2460                   "There was already a child by that name, and you asked me "
2461                   "to not replace it")
2462         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2463         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2464         return d
2465
2466     def test_POST_mkdir_no_replace_field(self): # return value?
2467         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2468                       replace="false")
2469         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2470                   "409 Conflict",
2471                   "There was already a child by that name, and you asked me "
2472                   "to not replace it")
2473         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2474         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2475         return d
2476
2477     def test_POST_mkdir_whendone_field(self):
2478         d = self.POST(self.public_url + "/foo",
2479                       t="mkdir", name="newdir", when_done="/THERE")
2480         d.addBoth(self.shouldRedirect, "/THERE")
2481         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2482         d.addCallback(self.failUnlessNodeKeysAre, [])
2483         return d
2484
2485     def test_POST_mkdir_whendone_queryarg(self):
2486         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2487                       t="mkdir", name="newdir")
2488         d.addBoth(self.shouldRedirect, "/THERE")
2489         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2490         d.addCallback(self.failUnlessNodeKeysAre, [])
2491         return d
2492
2493     def test_POST_bad_t(self):
2494         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2495                              "POST to a directory with bad t=BOGUS",
2496                              self.POST, self.public_url + "/foo", t="BOGUS")
2497         return d
2498
2499     def test_POST_set_children(self, command_name="set_children"):
2500         contents9, n9, newuri9 = self.makefile(9)
2501         contents10, n10, newuri10 = self.makefile(10)
2502         contents11, n11, newuri11 = self.makefile(11)
2503
2504         reqbody = """{
2505                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2506                                                 "size": 0,
2507                                                 "metadata": {
2508                                                   "ctime": 1002777696.7564139,
2509                                                   "mtime": 1002777696.7564139
2510                                                  }
2511                                                } ],
2512                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2513                                                 "size": 1,
2514                                                 "metadata": {
2515                                                   "ctime": 1002777696.7564139,
2516                                                   "mtime": 1002777696.7564139
2517                                                  }
2518                                                } ],
2519                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2520                                                 "size": 2,
2521                                                 "metadata": {
2522                                                   "ctime": 1002777696.7564139,
2523                                                   "mtime": 1002777696.7564139
2524                                                  }
2525                                                } ]
2526                     }""" % (newuri9, newuri10, newuri11)
2527
2528         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2529
2530         d = client.getPage(url, method="POST", postdata=reqbody)
2531         def _then(res):
2532             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2533             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2534             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2535
2536         d.addCallback(_then)
2537         d.addErrback(self.dump_error)
2538         return d
2539
2540     def test_POST_set_children_with_hyphen(self):
2541         return self.test_POST_set_children(command_name="set-children")
2542
2543     def test_POST_link_uri(self):
2544         contents, n, newuri = self.makefile(8)
2545         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2546         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2547         d.addCallback(lambda res:
2548                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2549                                                       contents))
2550         return d
2551
2552     def test_POST_link_uri_replace(self):
2553         contents, n, newuri = self.makefile(8)
2554         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2555         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2556         d.addCallback(lambda res:
2557                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2558                                                       contents))
2559         return d
2560
2561     def test_POST_link_uri_unknown_bad(self):
2562         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2563         d.addBoth(self.shouldFail, error.Error,
2564                   "POST_link_uri_unknown_bad",
2565                   "400 Bad Request",
2566                   "unknown cap in a write slot")
2567         return d
2568
2569     def test_POST_link_uri_unknown_ro_good(self):
2570         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2571         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2572         return d
2573
2574     def test_POST_link_uri_unknown_imm_good(self):
2575         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2576         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2577         return d
2578
2579     def test_POST_link_uri_no_replace_queryarg(self):
2580         contents, n, newuri = self.makefile(8)
2581         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2582                       name="bar.txt", uri=newuri)
2583         d.addBoth(self.shouldFail, error.Error,
2584                   "POST_link_uri_no_replace_queryarg",
2585                   "409 Conflict",
2586                   "There was already a child by that name, and you asked me "
2587                   "to not replace it")
2588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2589         d.addCallback(self.failUnlessIsBarDotTxt)
2590         return d
2591
2592     def test_POST_link_uri_no_replace_field(self):
2593         contents, n, newuri = self.makefile(8)
2594         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2595                       name="bar.txt", uri=newuri)
2596         d.addBoth(self.shouldFail, error.Error,
2597                   "POST_link_uri_no_replace_field",
2598                   "409 Conflict",
2599                   "There was already a child by that name, and you asked me "
2600                   "to not replace it")
2601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2602         d.addCallback(self.failUnlessIsBarDotTxt)
2603         return d
2604
2605     def test_POST_delete(self):
2606         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2607         d.addCallback(lambda res: self._foo_node.list())
2608         def _check(children):
2609             self.failIf(u"bar.txt" in children)
2610         d.addCallback(_check)
2611         return d
2612
2613     def test_POST_rename_file(self):
2614         d = self.POST(self.public_url + "/foo", t="rename",
2615                       from_name="bar.txt", to_name='wibble.txt')
2616         d.addCallback(lambda res:
2617                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2618         d.addCallback(lambda res:
2619                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2621         d.addCallback(self.failUnlessIsBarDotTxt)
2622         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2623         d.addCallback(self.failUnlessIsBarJSON)
2624         return d
2625
2626     def test_POST_rename_file_redundant(self):
2627         d = self.POST(self.public_url + "/foo", t="rename",
2628                       from_name="bar.txt", to_name='bar.txt')
2629         d.addCallback(lambda res:
2630                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2631         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2632         d.addCallback(self.failUnlessIsBarDotTxt)
2633         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2634         d.addCallback(self.failUnlessIsBarJSON)
2635         return d
2636
2637     def test_POST_rename_file_replace(self):
2638         # rename a file and replace a directory with it
2639         d = self.POST(self.public_url + "/foo", t="rename",
2640                       from_name="bar.txt", to_name='empty')
2641         d.addCallback(lambda res:
2642                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2643         d.addCallback(lambda res:
2644                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2645         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2646         d.addCallback(self.failUnlessIsBarDotTxt)
2647         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2648         d.addCallback(self.failUnlessIsBarJSON)
2649         return d
2650
2651     def test_POST_rename_file_no_replace_queryarg(self):
2652         # rename a file and replace a directory with it
2653         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2654                       from_name="bar.txt", to_name='empty')
2655         d.addBoth(self.shouldFail, error.Error,
2656                   "POST_rename_file_no_replace_queryarg",
2657                   "409 Conflict",
2658                   "There was already a child by that name, and you asked me "
2659                   "to not replace it")
2660         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2661         d.addCallback(self.failUnlessIsEmptyJSON)
2662         return d
2663
2664     def test_POST_rename_file_no_replace_field(self):
2665         # rename a file and replace a directory with it
2666         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2667                       from_name="bar.txt", to_name='empty')
2668         d.addBoth(self.shouldFail, error.Error,
2669                   "POST_rename_file_no_replace_field",
2670                   "409 Conflict",
2671                   "There was already a child by that name, and you asked me "
2672                   "to not replace it")
2673         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2674         d.addCallback(self.failUnlessIsEmptyJSON)
2675         return d
2676
2677     def failUnlessIsEmptyJSON(self, res):
2678         data = simplejson.loads(res)
2679         self.failUnlessEqual(data[0], "dirnode", data)
2680         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2681
2682     def test_POST_rename_file_slash_fail(self):
2683         d = self.POST(self.public_url + "/foo", t="rename",
2684                       from_name="bar.txt", to_name='kirk/spock.txt')
2685         d.addBoth(self.shouldFail, error.Error,
2686                   "test_POST_rename_file_slash_fail",
2687                   "400 Bad Request",
2688                   "to_name= may not contain a slash",
2689                   )
2690         d.addCallback(lambda res:
2691                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2692         return d
2693
2694     def test_POST_rename_dir(self):
2695         d = self.POST(self.public_url, t="rename",
2696                       from_name="foo", to_name='plunk')
2697         d.addCallback(lambda res:
2698                       self.failIfNodeHasChild(self.public_root, u"foo"))
2699         d.addCallback(lambda res:
2700                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2701         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2702         d.addCallback(self.failUnlessIsFooJSON)
2703         return d
2704
2705     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2706         """ If target is not None then the redirection has to go to target.  If
2707         statuscode is not None then the redirection has to be accomplished with
2708         that HTTP status code."""
2709         if not isinstance(res, failure.Failure):
2710             to_where = (target is None) and "somewhere" or ("to " + target)
2711             self.fail("%s: we were expecting to get redirected %s, not get an"
2712                       " actual page: %s" % (which, to_where, res))
2713         res.trap(error.PageRedirect)
2714         if statuscode is not None:
2715             self.failUnlessReallyEqual(res.value.status, statuscode,
2716                                        "%s: not a redirect" % which)
2717         if target is not None:
2718             # the PageRedirect does not seem to capture the uri= query arg
2719             # properly, so we can't check for it.
2720             realtarget = self.webish_url + target
2721             self.failUnlessReallyEqual(res.value.location, realtarget,
2722                                        "%s: wrong target" % which)
2723         return res.value.location
2724
2725     def test_GET_URI_form(self):
2726         base = "/uri?uri=%s" % self._bar_txt_uri
2727         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2728         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2729         d = self.GET(base)
2730         d.addBoth(self.shouldRedirect, targetbase)
2731         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2732         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2733         d.addCallback(lambda res: self.GET(base+"&t=json"))
2734         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2735         d.addCallback(self.log, "about to get file by uri")
2736         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2737         d.addCallback(self.failUnlessIsBarDotTxt)
2738         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2739         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2740                                            followRedirect=True))
2741         d.addCallback(self.failUnlessIsFooJSON)
2742         d.addCallback(self.log, "got dir by uri")
2743
2744         return d
2745
2746     def test_GET_URI_form_bad(self):
2747         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2748                              "400 Bad Request", "GET /uri requires uri=",
2749                              self.GET, "/uri")
2750         return d
2751
2752     def test_GET_rename_form(self):
2753         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2754                      followRedirect=True)
2755         def _check(res):
2756             self.failUnless('name="when_done" value="."' in res, res)
2757             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2758         d.addCallback(_check)
2759         return d
2760
2761     def log(self, res, msg):
2762         #print "MSG: %s  RES: %s" % (msg, res)
2763         log.msg(msg)
2764         return res
2765
2766     def test_GET_URI_URL(self):
2767         base = "/uri/%s" % self._bar_txt_uri
2768         d = self.GET(base)
2769         d.addCallback(self.failUnlessIsBarDotTxt)
2770         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2771         d.addCallback(self.failUnlessIsBarDotTxt)
2772         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2773         d.addCallback(self.failUnlessIsBarDotTxt)
2774         return d
2775
2776     def test_GET_URI_URL_dir(self):
2777         base = "/uri/%s?t=json" % self._foo_uri
2778         d = self.GET(base)
2779         d.addCallback(self.failUnlessIsFooJSON)
2780         return d
2781
2782     def test_GET_URI_URL_missing(self):
2783         base = "/uri/%s" % self._bad_file_uri
2784         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2785                                  http.GONE, None, "NotEnoughSharesError",
2786                                  self.GET, base)
2787         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2788         # here? we must arrange for a download to fail after target.open()
2789         # has been called, and then inspect the response to see that it is
2790         # shorter than we expected.
2791         return d
2792
2793     def test_PUT_DIRURL_uri(self):
2794         d = self.s.create_dirnode()
2795         def _made_dir(dn):
2796             new_uri = dn.get_uri()
2797             # replace /foo with a new (empty) directory
2798             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2799             d.addCallback(lambda res:
2800                           self.failUnlessReallyEqual(res.strip(), new_uri))
2801             d.addCallback(lambda res:
2802                           self.failUnlessRWChildURIIs(self.public_root,
2803                                                       u"foo",
2804                                                       new_uri))
2805             return d
2806         d.addCallback(_made_dir)
2807         return d
2808
2809     def test_PUT_DIRURL_uri_noreplace(self):
2810         d = self.s.create_dirnode()
2811         def _made_dir(dn):
2812             new_uri = dn.get_uri()
2813             # replace /foo with a new (empty) directory, but ask that
2814             # replace=false, so it should fail
2815             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2816                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2817                                  self.PUT,
2818                                  self.public_url + "/foo?t=uri&replace=false",
2819                                  new_uri)
2820             d.addCallback(lambda res:
2821                           self.failUnlessRWChildURIIs(self.public_root,
2822                                                       u"foo",
2823                                                       self._foo_uri))
2824             return d
2825         d.addCallback(_made_dir)
2826         return d
2827
2828     def test_PUT_DIRURL_bad_t(self):
2829         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2830                                  "400 Bad Request", "PUT to a directory",
2831                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2832         d.addCallback(lambda res:
2833                       self.failUnlessRWChildURIIs(self.public_root,
2834                                                   u"foo",
2835                                                   self._foo_uri))
2836         return d
2837
2838     def test_PUT_NEWFILEURL_uri(self):
2839         contents, n, new_uri = self.makefile(8)
2840         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2841         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2842         d.addCallback(lambda res:
2843                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2844                                                       contents))
2845         return d
2846
2847     def test_PUT_NEWFILEURL_uri_replace(self):
2848         contents, n, new_uri = self.makefile(8)
2849         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2850         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2851         d.addCallback(lambda res:
2852                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2853                                                       contents))
2854         return d
2855
2856     def test_PUT_NEWFILEURL_uri_no_replace(self):
2857         contents, n, new_uri = self.makefile(8)
2858         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2859         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2860                   "409 Conflict",
2861                   "There was already a child by that name, and you asked me "
2862                   "to not replace it")
2863         return d
2864
2865     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2866         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2867         d.addBoth(self.shouldFail, error.Error,
2868                   "POST_put_uri_unknown_bad",
2869                   "400 Bad Request",
2870                   "unknown cap in a write slot")
2871         return d
2872
2873     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2874         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2875         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2876                       u"put-future-ro.txt")
2877         return d
2878
2879     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2880         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2881         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2882                       u"put-future-imm.txt")
2883         return d
2884
2885     def test_PUT_NEWFILE_URI(self):
2886         file_contents = "New file contents here\n"
2887         d = self.PUT("/uri", file_contents)
2888         def _check(uri):
2889             assert isinstance(uri, str), uri
2890             self.failUnless(uri in FakeCHKFileNode.all_contents)
2891             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2892                                        file_contents)
2893             return self.GET("/uri/%s" % uri)
2894         d.addCallback(_check)
2895         def _check2(res):
2896             self.failUnlessReallyEqual(res, file_contents)
2897         d.addCallback(_check2)
2898         return d
2899
2900     def test_PUT_NEWFILE_URI_not_mutable(self):
2901         file_contents = "New file contents here\n"
2902         d = self.PUT("/uri?mutable=false", file_contents)
2903         def _check(uri):
2904             assert isinstance(uri, str), uri
2905             self.failUnless(uri in FakeCHKFileNode.all_contents)
2906             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2907                                        file_contents)
2908             return self.GET("/uri/%s" % uri)
2909         d.addCallback(_check)
2910         def _check2(res):
2911             self.failUnlessReallyEqual(res, file_contents)
2912         d.addCallback(_check2)
2913         return d
2914
2915     def test_PUT_NEWFILE_URI_only_PUT(self):
2916         d = self.PUT("/uri?t=bogus", "")
2917         d.addBoth(self.shouldFail, error.Error,
2918                   "PUT_NEWFILE_URI_only_PUT",
2919                   "400 Bad Request",
2920                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2921         return d
2922
2923     def test_PUT_NEWFILE_URI_mutable(self):
2924         file_contents = "New file contents here\n"
2925         d = self.PUT("/uri?mutable=true", file_contents)
2926         def _check1(filecap):
2927             filecap = filecap.strip()
2928             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2929             self.filecap = filecap
2930             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2931             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2932             n = self.s.create_node_from_uri(filecap)
2933             return n.download_best_version()
2934         d.addCallback(_check1)
2935         def _check2(data):
2936             self.failUnlessReallyEqual(data, file_contents)
2937             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2938         d.addCallback(_check2)
2939         def _check3(res):
2940             self.failUnlessReallyEqual(res, file_contents)
2941         d.addCallback(_check3)
2942         return d
2943
2944     def test_PUT_mkdir(self):
2945         d = self.PUT("/uri?t=mkdir", "")
2946         def _check(uri):
2947             n = self.s.create_node_from_uri(uri.strip())
2948             d2 = self.failUnlessNodeKeysAre(n, [])
2949             d2.addCallback(lambda res:
2950                            self.GET("/uri/%s?t=json" % uri))
2951             return d2
2952         d.addCallback(_check)
2953         d.addCallback(self.failUnlessIsEmptyJSON)
2954         return d
2955
2956     def test_POST_check(self):
2957         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2958         def _done(res):
2959             # this returns a string form of the results, which are probably
2960             # None since we're using fake filenodes.
2961             # TODO: verify that the check actually happened, by changing
2962             # FakeCHKFileNode to count how many times .check() has been
2963             # called.
2964             pass
2965         d.addCallback(_done)
2966         return d
2967
2968     def test_bad_method(self):
2969         url = self.webish_url + self.public_url + "/foo/bar.txt"
2970         d = self.shouldHTTPError("test_bad_method",
2971                                  501, "Not Implemented",
2972                                  "I don't know how to treat a BOGUS request.",
2973                                  client.getPage, url, method="BOGUS")
2974         return d
2975
2976     def test_short_url(self):
2977         url = self.webish_url + "/uri"
2978         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2979                                  "I don't know how to treat a DELETE request.",
2980                                  client.getPage, url, method="DELETE")
2981         return d
2982
2983     def test_ophandle_bad(self):
2984         url = self.webish_url + "/operations/bogus?t=status"
2985         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2986                                  "unknown/expired handle 'bogus'",
2987                                  client.getPage, url)
2988         return d
2989
2990     def test_ophandle_cancel(self):
2991         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2992                       followRedirect=True)
2993         d.addCallback(lambda ignored:
2994                       self.GET("/operations/128?t=status&output=JSON"))
2995         def _check1(res):
2996             data = simplejson.loads(res)
2997             self.failUnless("finished" in data, res)
2998             monitor = self.ws.root.child_operations.handles["128"][0]
2999             d = self.POST("/operations/128?t=cancel&output=JSON")
3000             def _check2(res):
3001                 data = simplejson.loads(res)
3002                 self.failUnless("finished" in data, res)
3003                 # t=cancel causes the handle to be forgotten
3004                 self.failUnless(monitor.is_cancelled())
3005             d.addCallback(_check2)
3006             return d
3007         d.addCallback(_check1)
3008         d.addCallback(lambda ignored:
3009                       self.shouldHTTPError("test_ophandle_cancel",
3010                                            404, "404 Not Found",
3011                                            "unknown/expired handle '128'",
3012                                            self.GET,
3013                                            "/operations/128?t=status&output=JSON"))
3014         return d
3015
3016     def test_ophandle_retainfor(self):
3017         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3018                       followRedirect=True)
3019         d.addCallback(lambda ignored:
3020                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3021         def _check1(res):
3022             data = simplejson.loads(res)
3023             self.failUnless("finished" in data, res)
3024         d.addCallback(_check1)
3025         # the retain-for=0 will cause the handle to be expired very soon
3026         d.addCallback(lambda ign:
3027             self.clock.advance(2.0))
3028         d.addCallback(lambda ignored:
3029                       self.shouldHTTPError("test_ophandle_retainfor",
3030                                            404, "404 Not Found",
3031                                            "unknown/expired handle '129'",
3032                                            self.GET,
3033                                            "/operations/129?t=status&output=JSON"))
3034         return d
3035
3036     def test_ophandle_release_after_complete(self):
3037         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3038                       followRedirect=True)
3039         d.addCallback(self.wait_for_operation, "130")
3040         d.addCallback(lambda ignored:
3041                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3042         # the release-after-complete=true will cause the handle to be expired
3043         d.addCallback(lambda ignored:
3044                       self.shouldHTTPError("test_ophandle_release_after_complete",
3045                                            404, "404 Not Found",
3046                                            "unknown/expired handle '130'",
3047                                            self.GET,
3048                                            "/operations/130?t=status&output=JSON"))
3049         return d
3050
3051     def test_uncollected_ophandle_expiration(self):
3052         # uncollected ophandles should expire after 4 days
3053         def _make_uncollected_ophandle(ophandle):
3054             d = self.POST(self.public_url +
3055                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3056                           followRedirect=False)
3057             # When we start the operation, the webapi server will want
3058             # to redirect us to the page for the ophandle, so we get
3059             # confirmation that the operation has started. If the
3060             # manifest operation has finished by the time we get there,
3061             # following that redirect (by setting followRedirect=True
3062             # above) has the side effect of collecting the ophandle that
3063             # we've just created, which means that we can't use the
3064             # ophandle to test the uncollected timeout anymore. So,
3065             # instead, catch the 302 here and don't follow it.
3066             d.addBoth(self.should302, "uncollected_ophandle_creation")
3067             return d
3068         # Create an ophandle, don't collect it, then advance the clock by
3069         # 4 days - 1 second and make sure that the ophandle is still there.
3070         d = _make_uncollected_ophandle(131)
3071         d.addCallback(lambda ign:
3072             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3073         d.addCallback(lambda ign:
3074             self.GET("/operations/131?t=status&output=JSON"))
3075         def _check1(res):
3076             data = simplejson.loads(res)
3077             self.failUnless("finished" in data, res)
3078         d.addCallback(_check1)
3079         # Create an ophandle, don't collect it, then try to collect it
3080         # after 4 days. It should be gone.
3081         d.addCallback(lambda ign:
3082             _make_uncollected_ophandle(132))
3083         d.addCallback(lambda ign:
3084             self.clock.advance(96*60*60))
3085         d.addCallback(lambda ign:
3086             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3087                                  404, "404 Not Found",
3088                                  "unknown/expired handle '132'",
3089                                  self.GET,
3090                                  "/operations/132?t=status&output=JSON"))
3091         return d
3092
3093     def test_collected_ophandle_expiration(self):
3094         # collected ophandles should expire after 1 day
3095         def _make_collected_ophandle(ophandle):
3096             d = self.POST(self.public_url +
3097                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3098                           followRedirect=True)
3099             # By following the initial redirect, we collect the ophandle
3100             # we've just created.
3101             return d
3102         # Create a collected ophandle, then collect it after 23 hours
3103         # and 59 seconds to make sure that it is still there.
3104         d = _make_collected_ophandle(133)
3105         d.addCallback(lambda ign:
3106             self.clock.advance((24*60*60) - 1))
3107         d.addCallback(lambda ign:
3108             self.GET("/operations/133?t=status&output=JSON"))
3109         def _check1(res):
3110             data = simplejson.loads(res)
3111             self.failUnless("finished" in data, res)
3112         d.addCallback(_check1)
3113         # Create another uncollected ophandle, then try to collect it
3114         # after 24 hours to make sure that it is gone.
3115         d.addCallback(lambda ign:
3116             _make_collected_ophandle(134))
3117         d.addCallback(lambda ign:
3118             self.clock.advance(24*60*60))
3119         d.addCallback(lambda ign:
3120             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3121                                  404, "404 Not Found",
3122                                  "unknown/expired handle '134'",
3123                                  self.GET,
3124                                  "/operations/134?t=status&output=JSON"))
3125         return d
3126
3127     def test_incident(self):
3128         d = self.POST("/report_incident", details="eek")
3129         def _done(res):
3130             self.failUnless("Thank you for your report!" in res, res)
3131         d.addCallback(_done)
3132         return d
3133
3134     def test_static(self):
3135         webdir = os.path.join(self.staticdir, "subdir")
3136         fileutil.make_dirs(webdir)
3137         f = open(os.path.join(webdir, "hello.txt"), "wb")
3138         f.write("hello")
3139         f.close()
3140
3141         d = self.GET("/static/subdir/hello.txt")
3142         def _check(res):
3143             self.failUnlessReallyEqual(res, "hello")
3144         d.addCallback(_check)
3145         return d
3146
3147
3148 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3149     def test_load_file(self):
3150         # This will raise an exception unless a well-formed XML file is found under that name.
3151         common.getxmlfile('directory.xhtml').load()
3152
3153     def test_parse_replace_arg(self):
3154         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3155         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3156         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3157                                    "only-files")
3158         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3159                         common.parse_replace_arg, "only_fles")
3160
3161     def test_abbreviate_time(self):
3162         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3163         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3164         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3165         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3166         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3167         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3168
3169     def test_abbreviate_rate(self):
3170         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3171         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3172         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3173         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3174
3175     def test_abbreviate_size(self):
3176         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3177         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3178         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3179         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3180         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3181
3182     def test_plural(self):
3183         def convert(s):
3184             return "%d second%s" % (s, status.plural(s))
3185         self.failUnlessReallyEqual(convert(0), "0 seconds")
3186         self.failUnlessReallyEqual(convert(1), "1 second")
3187         self.failUnlessReallyEqual(convert(2), "2 seconds")
3188         def convert2(s):
3189             return "has share%s: %s" % (status.plural(s), ",".join(s))
3190         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3191         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3192         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3193
3194
3195 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3196
3197     def CHECK(self, ign, which, args, clientnum=0):
3198         fileurl = self.fileurls[which]
3199         url = fileurl + "?" + args
3200         return self.GET(url, method="POST", clientnum=clientnum)
3201
3202     def test_filecheck(self):
3203         self.basedir = "web/Grid/filecheck"
3204         self.set_up_grid()
3205         c0 = self.g.clients[0]
3206         self.uris = {}
3207         DATA = "data" * 100
3208         d = c0.upload(upload.Data(DATA, convergence=""))
3209         def _stash_uri(ur, which):
3210             self.uris[which] = ur.uri
3211         d.addCallback(_stash_uri, "good")
3212         d.addCallback(lambda ign:
3213                       c0.upload(upload.Data(DATA+"1", convergence="")))
3214         d.addCallback(_stash_uri, "sick")
3215         d.addCallback(lambda ign:
3216                       c0.upload(upload.Data(DATA+"2", convergence="")))
3217         d.addCallback(_stash_uri, "dead")
3218         def _stash_mutable_uri(n, which):
3219             self.uris[which] = n.get_uri()
3220             assert isinstance(self.uris[which], str)
3221         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3222         d.addCallback(_stash_mutable_uri, "corrupt")
3223         d.addCallback(lambda ign:
3224                       c0.upload(upload.Data("literal", convergence="")))
3225         d.addCallback(_stash_uri, "small")
3226         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3227         d.addCallback(_stash_mutable_uri, "smalldir")
3228
3229         def _compute_fileurls(ignored):
3230             self.fileurls = {}
3231             for which in self.uris:
3232                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3233         d.addCallback(_compute_fileurls)
3234
3235         def _clobber_shares(ignored):
3236             good_shares = self.find_uri_shares(self.uris["good"])
3237             self.failUnlessReallyEqual(len(good_shares), 10)
3238             sick_shares = self.find_uri_shares(self.uris["sick"])
3239             os.unlink(sick_shares[0][2])
3240             dead_shares = self.find_uri_shares(self.uris["dead"])
3241             for i in range(1, 10):
3242                 os.unlink(dead_shares[i][2])
3243             c_shares = self.find_uri_shares(self.uris["corrupt"])
3244             cso = CorruptShareOptions()
3245             cso.stdout = StringIO()
3246             cso.parseOptions([c_shares[0][2]])
3247             corrupt_share(cso)
3248         d.addCallback(_clobber_shares)
3249
3250         d.addCallback(self.CHECK, "good", "t=check")
3251         def _got_html_good(res):
3252             self.failUnless("Healthy" in res, res)
3253             self.failIf("Not Healthy" in res, res)
3254         d.addCallback(_got_html_good)
3255         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3256         def _got_html_good_return_to(res):
3257             self.failUnless("Healthy" in res, res)
3258             self.failIf("Not Healthy" in res, res)
3259             self.failUnless('<a href="somewhere">Return to file'
3260                             in res, res)
3261         d.addCallback(_got_html_good_return_to)
3262         d.addCallback(self.CHECK, "good", "t=check&output=json")
3263         def _got_json_good(res):
3264             r = simplejson.loads(res)
3265             self.failUnlessEqual(r["summary"], "Healthy")
3266             self.failUnless(r["results"]["healthy"])
3267             self.failIf(r["results"]["needs-rebalancing"])
3268             self.failUnless(r["results"]["recoverable"])
3269         d.addCallback(_got_json_good)
3270
3271         d.addCallback(self.CHECK, "small", "t=check")
3272         def _got_html_small(res):
3273             self.failUnless("Literal files are always healthy" in res, res)
3274             self.failIf("Not Healthy" in res, res)
3275         d.addCallback(_got_html_small)
3276         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3277         def _got_html_small_return_to(res):
3278             self.failUnless("Literal files are always healthy" in res, res)
3279             self.failIf("Not Healthy" in res, res)
3280             self.failUnless('<a href="somewhere">Return to file'
3281                             in res, res)
3282         d.addCallback(_got_html_small_return_to)
3283         d.addCallback(self.CHECK, "small", "t=check&output=json")
3284         def _got_json_small(res):
3285             r = simplejson.loads(res)
3286             self.failUnlessEqual(r["storage-index"], "")
3287             self.failUnless(r["results"]["healthy"])
3288         d.addCallback(_got_json_small)
3289
3290         d.addCallback(self.CHECK, "smalldir", "t=check")
3291         def _got_html_smalldir(res):
3292             self.failUnless("Literal files are always healthy" in res, res)
3293             self.failIf("Not Healthy" in res, res)
3294         d.addCallback(_got_html_smalldir)
3295         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3296         def _got_json_smalldir(res):
3297             r = simplejson.loads(res)
3298             self.failUnlessEqual(r["storage-index"], "")
3299             self.failUnless(r["results"]["healthy"])
3300         d.addCallback(_got_json_smalldir)
3301
3302         d.addCallback(self.CHECK, "sick", "t=check")
3303         def _got_html_sick(res):
3304             self.failUnless("Not Healthy" in res, res)
3305         d.addCallback(_got_html_sick)
3306         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3307         def _got_json_sick(res):
3308             r = simplejson.loads(res)
3309             self.failUnlessEqual(r["summary"],
3310                                  "Not Healthy: 9 shares (enc 3-of-10)")
3311             self.failIf(r["results"]["healthy"])
3312             self.failIf(r["results"]["needs-rebalancing"])
3313             self.failUnless(r["results"]["recoverable"])
3314         d.addCallback(_got_json_sick)
3315
3316         d.addCallback(self.CHECK, "dead", "t=check")
3317         def _got_html_dead(res):
3318             self.failUnless("Not Healthy" in res, res)
3319         d.addCallback(_got_html_dead)
3320         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3321         def _got_json_dead(res):
3322             r = simplejson.loads(res)
3323             self.failUnlessEqual(r["summary"],
3324                                  "Not Healthy: 1 shares (enc 3-of-10)")
3325             self.failIf(r["results"]["healthy"])
3326             self.failIf(r["results"]["needs-rebalancing"])
3327             self.failIf(r["results"]["recoverable"])
3328         d.addCallback(_got_json_dead)
3329
3330         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3331         def _got_html_corrupt(res):
3332             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3333         d.addCallback(_got_html_corrupt)
3334         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3335         def _got_json_corrupt(res):
3336             r = simplejson.loads(res)
3337             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3338                             r["summary"])
3339             self.failIf(r["results"]["healthy"])
3340             self.failUnless(r["results"]["recoverable"])
3341             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3342             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3343         d.addCallback(_got_json_corrupt)
3344
3345         d.addErrback(self.explain_web_error)
3346         return d
3347
3348     def test_repair_html(self):
3349         self.basedir = "web/Grid/repair_html"
3350         self.set_up_grid()
3351         c0 = self.g.clients[0]
3352         self.uris = {}
3353         DATA = "data" * 100
3354         d = c0.upload(upload.Data(DATA, convergence=""))
3355         def _stash_uri(ur, which):
3356             self.uris[which] = ur.uri
3357         d.addCallback(_stash_uri, "good")
3358         d.addCallback(lambda ign:
3359                       c0.upload(upload.Data(DATA+"1", convergence="")))
3360         d.addCallback(_stash_uri, "sick")
3361         d.addCallback(lambda ign:
3362                       c0.upload(upload.Data(DATA+"2", convergence="")))
3363         d.addCallback(_stash_uri, "dead")
3364         def _stash_mutable_uri(n, which):
3365             self.uris[which] = n.get_uri()
3366             assert isinstance(self.uris[which], str)
3367         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3368         d.addCallback(_stash_mutable_uri, "corrupt")
3369
3370         def _compute_fileurls(ignored):
3371             self.fileurls = {}
3372             for which in self.uris:
3373                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3374         d.addCallback(_compute_fileurls)
3375
3376         def _clobber_shares(ignored):
3377             good_shares = self.find_uri_shares(self.uris["good"])
3378             self.failUnlessReallyEqual(len(good_shares), 10)
3379             sick_shares = self.find_uri_shares(self.uris["sick"])
3380             os.unlink(sick_shares[0][2])
3381             dead_shares = self.find_uri_shares(self.uris["dead"])
3382             for i in range(1, 10):
3383                 os.unlink(dead_shares[i][2])
3384             c_shares = self.find_uri_shares(self.uris["corrupt"])
3385             cso = CorruptShareOptions()
3386             cso.stdout = StringIO()
3387             cso.parseOptions([c_shares[0][2]])
3388             corrupt_share(cso)
3389         d.addCallback(_clobber_shares)
3390
3391         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3392         def _got_html_good(res):
3393             self.failUnless("Healthy" in res, res)
3394             self.failIf("Not Healthy" in res, res)
3395             self.failUnless("No repair necessary" in res, res)
3396         d.addCallback(_got_html_good)
3397
3398         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3399         def _got_html_sick(res):
3400             self.failUnless("Healthy : healthy" in res, res)
3401             self.failIf("Not Healthy" in res, res)
3402             self.failUnless("Repair successful" in res, res)
3403         d.addCallback(_got_html_sick)
3404
3405         # repair of a dead file will fail, of course, but it isn't yet
3406         # clear how this should be reported. Right now it shows up as
3407         # a "410 Gone".
3408         #
3409         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3410         #def _got_html_dead(res):
3411         #    print res
3412         #    self.failUnless("Healthy : healthy" in res, res)
3413         #    self.failIf("Not Healthy" in res, res)
3414         #    self.failUnless("No repair necessary" in res, res)
3415         #d.addCallback(_got_html_dead)
3416
3417         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3418         def _got_html_corrupt(res):
3419             self.failUnless("Healthy : Healthy" in res, res)
3420             self.failIf("Not Healthy" in res, res)
3421             self.failUnless("Repair successful" in res, res)
3422         d.addCallback(_got_html_corrupt)
3423
3424         d.addErrback(self.explain_web_error)
3425         return d
3426
3427     def test_repair_json(self):
3428         self.basedir = "web/Grid/repair_json"
3429         self.set_up_grid()
3430         c0 = self.g.clients[0]
3431         self.uris = {}
3432         DATA = "data" * 100
3433         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3434         def _stash_uri(ur, which):
3435             self.uris[which] = ur.uri
3436         d.addCallback(_stash_uri, "sick")
3437
3438         def _compute_fileurls(ignored):
3439             self.fileurls = {}
3440             for which in self.uris:
3441                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3442         d.addCallback(_compute_fileurls)
3443
3444         def _clobber_shares(ignored):
3445             sick_shares = self.find_uri_shares(self.uris["sick"])
3446             os.unlink(sick_shares[0][2])
3447         d.addCallback(_clobber_shares)
3448
3449         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3450         def _got_json_sick(res):
3451             r = simplejson.loads(res)
3452             self.failUnlessReallyEqual(r["repair-attempted"], True)
3453             self.failUnlessReallyEqual(r["repair-successful"], True)
3454             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3455                                  "Not Healthy: 9 shares (enc 3-of-10)")
3456             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3457             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3458             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3459         d.addCallback(_got_json_sick)
3460
3461         d.addErrback(self.explain_web_error)
3462         return d
3463
3464     def test_unknown(self, immutable=False):
3465         self.basedir = "web/Grid/unknown"
3466         if immutable:
3467             self.basedir = "web/Grid/unknown-immutable"
3468
3469         self.set_up_grid()
3470         c0 = self.g.clients[0]
3471         self.uris = {}
3472         self.fileurls = {}
3473
3474         # the future cap format may contain slashes, which must be tolerated
3475         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3476                                                            safe="")
3477
3478         if immutable:
3479             name = u"future-imm"
3480             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3481             d = c0.create_immutable_dirnode({name: (future_node, {})})
3482         else:
3483             name = u"future"
3484             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3485             d = c0.create_dirnode()
3486
3487         def _stash_root_and_create_file(n):
3488             self.rootnode = n
3489             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3490             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3491             if not immutable:
3492                 return self.rootnode.set_node(name, future_node)
3493         d.addCallback(_stash_root_and_create_file)
3494
3495         # make sure directory listing tolerates unknown nodes
3496         d.addCallback(lambda ign: self.GET(self.rooturl))
3497         def _check_directory_html(res, expected_type_suffix):
3498             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3499                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3500                                  re.DOTALL)
3501             self.failUnless(re.search(pattern, res), res)
3502             # find the More Info link for name, should be relative
3503             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3504             info_url = mo.group(1)
3505             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3506         if immutable:
3507             d.addCallback(_check_directory_html, "-IMM")
3508         else:
3509             d.addCallback(_check_directory_html, "")
3510
3511         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3512         def _check_directory_json(res, expect_rw_uri):
3513             data = simplejson.loads(res)
3514             self.failUnlessEqual(data[0], "dirnode")
3515             f = data[1]["children"][name]
3516             self.failUnlessEqual(f[0], "unknown")
3517             if expect_rw_uri:
3518                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
3519             else:
3520                 self.failIfIn("rw_uri", f[1])
3521             if immutable:
3522                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
3523             else:
3524                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
3525             self.failUnless("metadata" in f[1])
3526         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3527
3528         def _check_info(res, expect_rw_uri, expect_ro_uri):
3529             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3530             if expect_rw_uri:
3531                 self.failUnlessIn(unknown_rwcap, res)
3532             if expect_ro_uri:
3533                 if immutable:
3534                     self.failUnlessIn(unknown_immcap, res)
3535                 else:
3536                     self.failUnlessIn(unknown_rocap, res)
3537             else:
3538                 self.failIfIn(unknown_rocap, res)
3539             self.failIfIn("Raw data as", res)
3540             self.failIfIn("Directory writecap", res)
3541             self.failIfIn("Checker Operations", res)
3542             self.failIfIn("Mutable File Operations", res)
3543             self.failIfIn("Directory Operations", res)
3544
3545         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3546         # why they fail. Possibly related to ticket #922.
3547
3548         d.addCallback(lambda ign: self.GET(expected_info_url))
3549         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3550         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3551         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3552
3553         def _check_json(res, expect_rw_uri):
3554             data = simplejson.loads(res)
3555             self.failUnlessEqual(data[0], "unknown")
3556             if expect_rw_uri:
3557                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
3558             else:
3559                 self.failIfIn("rw_uri", data[1])
3560
3561             if immutable:
3562                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
3563                 self.failUnlessReallyEqual(data[1]["mutable"], False)
3564             elif expect_rw_uri:
3565                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3566                 self.failUnlessReallyEqual(data[1]["mutable"], True)
3567             else:
3568                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3569                 self.failIf("mutable" in data[1], data[1])
3570
3571             # TODO: check metadata contents
3572             self.failUnless("metadata" in data[1])
3573
3574         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3575         d.addCallback(_check_json, expect_rw_uri=not immutable)
3576
3577         # and make sure that a read-only version of the directory can be
3578         # rendered too. This version will not have unknown_rwcap, whether
3579         # or not future_node was immutable.
3580         d.addCallback(lambda ign: self.GET(self.rourl))
3581         if immutable:
3582             d.addCallback(_check_directory_html, "-IMM")
3583         else:
3584             d.addCallback(_check_directory_html, "-RO")
3585
3586         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3587         d.addCallback(_check_directory_json, expect_rw_uri=False)
3588
3589         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3590         d.addCallback(_check_json, expect_rw_uri=False)
3591
3592         # TODO: check that getting t=info from the Info link in the ro directory
3593         # works, and does not include the writecap URI.
3594         return d
3595
3596     def test_immutable_unknown(self):
3597         return self.test_unknown(immutable=True)
3598
3599     def test_mutant_dirnodes_are_omitted(self):
3600         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3601
3602         self.set_up_grid()
3603         c = self.g.clients[0]
3604         nm = c.nodemaker
3605         self.uris = {}
3606         self.fileurls = {}
3607
3608         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3609         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3610         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3611
3612         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3613         # test the dirnode and web layers separately.
3614
3615         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3616         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3617         # When the directory is read, the mutants should be silently disposed of, leaving
3618         # their lonely sibling.
3619         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3620         # because immutable directories don't have a writecap and therefore that field
3621         # isn't (and can't be) decrypted.
3622         # TODO: The field still exists in the netstring. Technically we should check what
3623         # happens if something is put there (_unpack_contents should raise ValueError),
3624         # but that can wait.
3625
3626         lonely_child = nm.create_from_cap(lonely_uri)
3627         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3628         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3629
3630         def _by_hook_or_by_crook():
3631             return True
3632         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3633             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3634
3635         mutant_write_in_ro_child.get_write_uri    = lambda: None
3636         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3637
3638         kids = {u"lonely":      (lonely_child, {}),
3639                 u"ro":          (mutant_ro_child, {}),
3640                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3641                 }
3642         d = c.create_immutable_dirnode(kids)
3643
3644         def _created(dn):
3645             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3646             self.failIf(dn.is_mutable())
3647             self.failUnless(dn.is_readonly())
3648             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3649             self.failIf(hasattr(dn._node, 'get_writekey'))
3650             rep = str(dn)
3651             self.failUnless("RO-IMM" in rep)
3652             cap = dn.get_cap()
3653             self.failUnlessIn("CHK", cap.to_string())
3654             self.cap = cap
3655             self.rootnode = dn
3656             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3657             return download_to_data(dn._node)
3658         d.addCallback(_created)
3659
3660         def _check_data(data):
3661             # Decode the netstring representation of the directory to check that all children
3662             # are present. This is a bit of an abstraction violation, but there's not really
3663             # any other way to do it given that the real DirectoryNode._unpack_contents would
3664             # strip the mutant children out (which is what we're trying to test, later).
3665             position = 0
3666             numkids = 0
3667             while position < len(data):
3668                 entries, position = split_netstring(data, 1, position)
3669                 entry = entries[0]
3670                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3671                 name = name_utf8.decode("utf-8")
3672                 self.failUnless(rwcapdata == "")
3673                 self.failUnless(name in kids)
3674                 (expected_child, ign) = kids[name]
3675                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3676                 numkids += 1
3677
3678             self.failUnlessReallyEqual(numkids, 3)
3679             return self.rootnode.list()
3680         d.addCallback(_check_data)
3681
3682         # Now when we use the real directory listing code, the mutants should be absent.
3683         def _check_kids(children):
3684             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3685             lonely_node, lonely_metadata = children[u"lonely"]
3686
3687             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3688             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3689         d.addCallback(_check_kids)
3690
3691         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3692         d.addCallback(lambda n: n.list())
3693         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3694
3695         # Make sure the lonely child can be listed in HTML...
3696         d.addCallback(lambda ign: self.GET(self.rooturl))
3697         def _check_html(res):
3698             self.failIfIn("URI:SSK", res)
3699             get_lonely = "".join([r'<td>FILE</td>',
3700                                   r'\s+<td>',
3701                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3702                                   r'</td>',
3703                                   r'\s+<td>%d</td>' % len("one"),
3704                                   ])
3705             self.failUnless(re.search(get_lonely, res), res)
3706
3707             # find the More Info link for name, should be relative
3708             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3709             info_url = mo.group(1)
3710             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3711         d.addCallback(_check_html)
3712
3713         # ... and in JSON.
3714         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3715         def _check_json(res):
3716             data = simplejson.loads(res)
3717             self.failUnlessEqual(data[0], "dirnode")
3718             listed_children = data[1]["children"]
3719             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3720             ll_type, ll_data = listed_children[u"lonely"]
3721             self.failUnlessEqual(ll_type, "filenode")
3722             self.failIf("rw_uri" in ll_data)
3723             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
3724         d.addCallback(_check_json)
3725         return d
3726
3727     def test_deep_check(self):
3728         self.basedir = "web/Grid/deep_check"
3729         self.set_up_grid()
3730         c0 = self.g.clients[0]
3731         self.uris = {}
3732         self.fileurls = {}
3733         DATA = "data" * 100
3734         d = c0.create_dirnode()
3735         def _stash_root_and_create_file(n):
3736             self.rootnode = n
3737             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3738             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3739         d.addCallback(_stash_root_and_create_file)
3740         def _stash_uri(fn, which):
3741             self.uris[which] = fn.get_uri()
3742             return fn
3743         d.addCallback(_stash_uri, "good")
3744         d.addCallback(lambda ign:
3745                       self.rootnode.add_file(u"small",
3746                                              upload.Data("literal",
3747                                                         convergence="")))
3748         d.addCallback(_stash_uri, "small")
3749         d.addCallback(lambda ign:
3750                       self.rootnode.add_file(u"sick",
3751                                              upload.Data(DATA+"1",
3752                                                         convergence="")))
3753         d.addCallback(_stash_uri, "sick")
3754
3755         # this tests that deep-check and stream-manifest will ignore
3756         # UnknownNode instances. Hopefully this will also cover deep-stats.
3757         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3758         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3759
3760         def _clobber_shares(ignored):
3761             self.delete_shares_numbered(self.uris["sick"], [0,1])
3762         d.addCallback(_clobber_shares)
3763
3764         # root
3765         # root/good
3766         # root/small
3767         # root/sick
3768         # root/future
3769
3770         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3771         def _done(res):
3772             try:
3773                 units = [simplejson.loads(line)
3774                          for line in res.splitlines()
3775                          if line]
3776             except ValueError:
3777                 print "response is:", res
3778                 print "undecodeable line was '%s'" % line
3779                 raise
3780             self.failUnlessReallyEqual(len(units), 5+1)
3781             # should be parent-first
3782             u0 = units[0]
3783             self.failUnlessEqual(u0["path"], [])
3784             self.failUnlessEqual(u0["type"], "directory")
3785             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3786             u0cr = u0["check-results"]
3787             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3788
3789             ugood = [u for u in units
3790                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3791             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
3792             ugoodcr = ugood["check-results"]
3793             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3794
3795             stats = units[-1]
3796             self.failUnlessEqual(stats["type"], "stats")
3797             s = stats["stats"]
3798             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3799             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3800             self.failUnlessReallyEqual(s["count-directories"], 1)
3801             self.failUnlessReallyEqual(s["count-unknown"], 1)
3802         d.addCallback(_done)
3803
3804         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3805         def _check_manifest(res):
3806             self.failUnless(res.endswith("\n"))
3807             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3808             self.failUnlessReallyEqual(len(units), 5+1)
3809             self.failUnlessEqual(units[-1]["type"], "stats")
3810             first = units[0]
3811             self.failUnlessEqual(first["path"], [])
3812             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
3813             self.failUnlessEqual(first["type"], "directory")
3814             stats = units[-1]["stats"]
3815             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3816             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3817             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3818             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3819             self.failUnlessReallyEqual(stats["count-unknown"], 1)
3820         d.addCallback(_check_manifest)
3821
3822         # now add root/subdir and root/subdir/grandchild, then make subdir
3823         # unrecoverable, then see what happens
3824
3825         d.addCallback(lambda ign:
3826                       self.rootnode.create_subdirectory(u"subdir"))
3827         d.addCallback(_stash_uri, "subdir")
3828         d.addCallback(lambda subdir_node:
3829                       subdir_node.add_file(u"grandchild",
3830                                            upload.Data(DATA+"2",
3831                                                        convergence="")))
3832         d.addCallback(_stash_uri, "grandchild")
3833
3834         d.addCallback(lambda ign:
3835                       self.delete_shares_numbered(self.uris["subdir"],
3836                                                   range(1, 10)))
3837
3838         # root
3839         # root/good
3840         # root/small
3841         # root/sick
3842         # root/future
3843         # root/subdir [unrecoverable]
3844         # root/subdir/grandchild
3845
3846         # how should a streaming-JSON API indicate fatal error?
3847         # answer: emit ERROR: instead of a JSON string
3848
3849         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3850         def _check_broken_manifest(res):
3851             lines = res.splitlines()
3852             error_lines = [i
3853                            for (i,line) in enumerate(lines)
3854                            if line.startswith("ERROR:")]
3855             if not error_lines:
3856                 self.fail("no ERROR: in output: %s" % (res,))
3857             first_error = error_lines[0]
3858             error_line = lines[first_error]
3859             error_msg = lines[first_error+1:]
3860             error_msg_s = "\n".join(error_msg) + "\n"
3861             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3862                               error_line)
3863             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3864             units = [simplejson.loads(line) for line in lines[:first_error]]
3865             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3866             last_unit = units[-1]
3867             self.failUnlessEqual(last_unit["path"], ["subdir"])
3868         d.addCallback(_check_broken_manifest)
3869
3870         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3871         def _check_broken_deepcheck(res):
3872             lines = res.splitlines()
3873             error_lines = [i
3874                            for (i,line) in enumerate(lines)
3875                            if line.startswith("ERROR:")]
3876             if not error_lines:
3877                 self.fail("no ERROR: in output: %s" % (res,))
3878             first_error = error_lines[0]
3879             error_line = lines[first_error]
3880             error_msg = lines[first_error+1:]
3881             error_msg_s = "\n".join(error_msg) + "\n"
3882             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3883                               error_line)
3884             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3885             units = [simplejson.loads(line) for line in lines[:first_error]]
3886             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3887             last_unit = units[-1]
3888             self.failUnlessEqual(last_unit["path"], ["subdir"])
3889             r = last_unit["check-results"]["results"]
3890             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3891             self.failUnlessReallyEqual(r["count-shares-good"], 1)
3892             self.failUnlessReallyEqual(r["recoverable"], False)
3893         d.addCallback(_check_broken_deepcheck)
3894
3895         d.addErrback(self.explain_web_error)
3896         return d
3897
3898     def test_deep_check_and_repair(self):
3899         self.basedir = "web/Grid/deep_check_and_repair"
3900         self.set_up_grid()
3901         c0 = self.g.clients[0]
3902         self.uris = {}
3903         self.fileurls = {}
3904         DATA = "data" * 100
3905         d = c0.create_dirnode()
3906         def _stash_root_and_create_file(n):
3907             self.rootnode = n
3908             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3909             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3910         d.addCallback(_stash_root_and_create_file)
3911         def _stash_uri(fn, which):
3912             self.uris[which] = fn.get_uri()
3913         d.addCallback(_stash_uri, "good")
3914         d.addCallback(lambda ign:
3915                       self.rootnode.add_file(u"small",
3916                                              upload.Data("literal",
3917                                                         convergence="")))
3918         d.addCallback(_stash_uri, "small")
3919         d.addCallback(lambda ign:
3920                       self.rootnode.add_file(u"sick",
3921                                              upload.Data(DATA+"1",
3922                                                         convergence="")))
3923         d.addCallback(_stash_uri, "sick")
3924         #d.addCallback(lambda ign:
3925         #              self.rootnode.add_file(u"dead",
3926         #                                     upload.Data(DATA+"2",
3927         #                                                convergence="")))
3928         #d.addCallback(_stash_uri, "dead")
3929
3930         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3931         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3932         #d.addCallback(_stash_uri, "corrupt")
3933
3934         def _clobber_shares(ignored):
3935             good_shares = self.find_uri_shares(self.uris["good"])
3936             self.failUnlessReallyEqual(len(good_shares), 10)
3937             sick_shares = self.find_uri_shares(self.uris["sick"])
3938             os.unlink(sick_shares[0][2])
3939             #dead_shares = self.find_uri_shares(self.uris["dead"])
3940             #for i in range(1, 10):
3941             #    os.unlink(dead_shares[i][2])
3942
3943             #c_shares = self.find_uri_shares(self.uris["corrupt"])
3944             #cso = CorruptShareOptions()
3945             #cso.stdout = StringIO()
3946             #cso.parseOptions([c_shares[0][2]])
3947             #corrupt_share(cso)
3948         d.addCallback(_clobber_shares)
3949
3950         # root
3951         # root/good   CHK, 10 shares
3952         # root/small  LIT
3953         # root/sick   CHK, 9 shares
3954
3955         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3956         def _done(res):
3957             units = [simplejson.loads(line)
3958                      for line in res.splitlines()
3959                      if line]
3960             self.failUnlessReallyEqual(len(units), 4+1)
3961             # should be parent-first
3962             u0 = units[0]
3963             self.failUnlessEqual(u0["path"], [])
3964             self.failUnlessEqual(u0["type"], "directory")
3965             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3966             u0crr = u0["check-and-repair-results"]
3967             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
3968             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3969
3970             ugood = [u for u in units
3971                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3972             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
3973             ugoodcrr = ugood["check-and-repair-results"]
3974             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
3975             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3976
3977             usick = [u for u in units
3978                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3979             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
3980             usickcrr = usick["check-and-repair-results"]
3981             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
3982             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
3983             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3984             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3985
3986             stats = units[-1]
3987             self.failUnlessEqual(stats["type"], "stats")
3988             s = stats["stats"]
3989             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3990             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3991             self.failUnlessReallyEqual(s["count-directories"], 1)
3992         d.addCallback(_done)
3993
3994         d.addErrback(self.explain_web_error)
3995         return d
3996
3997     def _count_leases(self, ignored, which):
3998         u = self.uris[which]
3999         shares = self.find_uri_shares(u)
4000         lease_counts = []
4001         for shnum, serverid, fn in shares:
4002             sf = get_share_file(fn)
4003             num_leases = len(list(sf.get_leases()))
4004         lease_counts.append( (fn, num_leases) )
4005         return lease_counts
4006
4007     def _assert_leasecount(self, lease_counts, expected):
4008         for (fn, num_leases) in lease_counts:
4009             if num_leases != expected:
4010                 self.fail("expected %d leases, have %d, on %s" %
4011                           (expected, num_leases, fn))
4012
4013     def test_add_lease(self):
4014         self.basedir = "web/Grid/add_lease"
4015         self.set_up_grid(num_clients=2)
4016         c0 = self.g.clients[0]
4017         self.uris = {}
4018         DATA = "data" * 100
4019         d = c0.upload(upload.Data(DATA, convergence=""))
4020         def _stash_uri(ur, which):
4021             self.uris[which] = ur.uri
4022         d.addCallback(_stash_uri, "one")
4023         d.addCallback(lambda ign:
4024                       c0.upload(upload.Data(DATA+"1", convergence="")))
4025         d.addCallback(_stash_uri, "two")
4026         def _stash_mutable_uri(n, which):
4027             self.uris[which] = n.get_uri()
4028             assert isinstance(self.uris[which], str)
4029         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
4030         d.addCallback(_stash_mutable_uri, "mutable")
4031
4032         def _compute_fileurls(ignored):
4033             self.fileurls = {}
4034             for which in self.uris:
4035                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4036         d.addCallback(_compute_fileurls)
4037
4038         d.addCallback(self._count_leases, "one")
4039         d.addCallback(self._assert_leasecount, 1)
4040         d.addCallback(self._count_leases, "two")
4041         d.addCallback(self._assert_leasecount, 1)
4042         d.addCallback(self._count_leases, "mutable")
4043         d.addCallback(self._assert_leasecount, 1)
4044
4045         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4046         def _got_html_good(res):
4047             self.failUnless("Healthy" in res, res)
4048             self.failIf("Not Healthy" in res, res)
4049         d.addCallback(_got_html_good)
4050
4051         d.addCallback(self._count_leases, "one")
4052         d.addCallback(self._assert_leasecount, 1)
4053         d.addCallback(self._count_leases, "two")
4054         d.addCallback(self._assert_leasecount, 1)
4055         d.addCallback(self._count_leases, "mutable")
4056         d.addCallback(self._assert_leasecount, 1)
4057
4058         # this CHECK uses the original client, which uses the same
4059         # lease-secrets, so it will just renew the original lease
4060         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4061         d.addCallback(_got_html_good)
4062
4063         d.addCallback(self._count_leases, "one")
4064         d.addCallback(self._assert_leasecount, 1)
4065         d.addCallback(self._count_leases, "two")
4066         d.addCallback(self._assert_leasecount, 1)
4067         d.addCallback(self._count_leases, "mutable")
4068         d.addCallback(self._assert_leasecount, 1)
4069
4070         # this CHECK uses an alternate client, which adds a second lease
4071         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4072         d.addCallback(_got_html_good)
4073
4074         d.addCallback(self._count_leases, "one")
4075         d.addCallback(self._assert_leasecount, 2)
4076         d.addCallback(self._count_leases, "two")
4077         d.addCallback(self._assert_leasecount, 1)
4078         d.addCallback(self._count_leases, "mutable")
4079         d.addCallback(self._assert_leasecount, 1)
4080
4081         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4082         d.addCallback(_got_html_good)
4083
4084         d.addCallback(self._count_leases, "one")
4085         d.addCallback(self._assert_leasecount, 2)
4086         d.addCallback(self._count_leases, "two")
4087         d.addCallback(self._assert_leasecount, 1)
4088         d.addCallback(self._count_leases, "mutable")
4089         d.addCallback(self._assert_leasecount, 1)
4090
4091         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4092                       clientnum=1)
4093         d.addCallback(_got_html_good)
4094
4095         d.addCallback(self._count_leases, "one")
4096         d.addCallback(self._assert_leasecount, 2)
4097         d.addCallback(self._count_leases, "two")
4098         d.addCallback(self._assert_leasecount, 1)
4099         d.addCallback(self._count_leases, "mutable")
4100         d.addCallback(self._assert_leasecount, 2)
4101
4102         d.addErrback(self.explain_web_error)
4103         return d
4104
4105     def test_deep_add_lease(self):
4106         self.basedir = "web/Grid/deep_add_lease"
4107         self.set_up_grid(num_clients=2)
4108         c0 = self.g.clients[0]
4109         self.uris = {}
4110         self.fileurls = {}
4111         DATA = "data" * 100
4112         d = c0.create_dirnode()
4113         def _stash_root_and_create_file(n):
4114             self.rootnode = n
4115             self.uris["root"] = n.get_uri()
4116             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4117             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4118         d.addCallback(_stash_root_and_create_file)
4119         def _stash_uri(fn, which):
4120             self.uris[which] = fn.get_uri()
4121         d.addCallback(_stash_uri, "one")
4122         d.addCallback(lambda ign:
4123                       self.rootnode.add_file(u"small",
4124                                              upload.Data("literal",
4125                                                         convergence="")))
4126         d.addCallback(_stash_uri, "small")
4127
4128         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4129         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4130         d.addCallback(_stash_uri, "mutable")
4131
4132         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4133         def _done(res):
4134             units = [simplejson.loads(line)
4135                      for line in res.splitlines()
4136                      if line]
4137             # root, one, small, mutable,   stats
4138             self.failUnlessReallyEqual(len(units), 4+1)
4139         d.addCallback(_done)
4140
4141         d.addCallback(self._count_leases, "root")
4142         d.addCallback(self._assert_leasecount, 1)
4143         d.addCallback(self._count_leases, "one")
4144         d.addCallback(self._assert_leasecount, 1)
4145         d.addCallback(self._count_leases, "mutable")
4146         d.addCallback(self._assert_leasecount, 1)
4147
4148         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4149         d.addCallback(_done)
4150
4151         d.addCallback(self._count_leases, "root")
4152         d.addCallback(self._assert_leasecount, 1)
4153         d.addCallback(self._count_leases, "one")
4154         d.addCallback(self._assert_leasecount, 1)
4155         d.addCallback(self._count_leases, "mutable")
4156         d.addCallback(self._assert_leasecount, 1)
4157
4158         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4159                       clientnum=1)
4160         d.addCallback(_done)
4161
4162         d.addCallback(self._count_leases, "root")
4163         d.addCallback(self._assert_leasecount, 2)
4164         d.addCallback(self._count_leases, "one")
4165         d.addCallback(self._assert_leasecount, 2)
4166         d.addCallback(self._count_leases, "mutable")
4167         d.addCallback(self._assert_leasecount, 2)
4168
4169         d.addErrback(self.explain_web_error)
4170         return d
4171
4172
4173     def test_exceptions(self):
4174         self.basedir = "web/Grid/exceptions"
4175         self.set_up_grid(num_clients=1, num_servers=2)
4176         c0 = self.g.clients[0]
4177         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4178         self.fileurls = {}
4179         DATA = "data" * 100
4180         d = c0.create_dirnode()
4181         def _stash_root(n):
4182             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4183             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4184             return n
4185         d.addCallback(_stash_root)
4186         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4187         def _stash_bad(ur):
4188             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4189             self.delete_shares_numbered(ur.uri, range(1,10))
4190
4191             u = uri.from_string(ur.uri)
4192             u.key = testutil.flip_bit(u.key, 0)
4193             baduri = u.to_string()
4194             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4195         d.addCallback(_stash_bad)
4196         d.addCallback(lambda ign: c0.create_dirnode())
4197         def _mangle_dirnode_1share(n):
4198             u = n.get_uri()
4199             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4200             self.fileurls["dir-1share-json"] = url + "?t=json"
4201             self.delete_shares_numbered(u, range(1,10))
4202         d.addCallback(_mangle_dirnode_1share)
4203         d.addCallback(lambda ign: c0.create_dirnode())
4204         def _mangle_dirnode_0share(n):
4205             u = n.get_uri()
4206             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4207             self.fileurls["dir-0share-json"] = url + "?t=json"
4208             self.delete_shares_numbered(u, range(0,10))
4209         d.addCallback(_mangle_dirnode_0share)
4210
4211         # NotEnoughSharesError should be reported sensibly, with a
4212         # text/plain explanation of the problem, and perhaps some
4213         # information on which shares *could* be found.
4214
4215         d.addCallback(lambda ignored:
4216                       self.shouldHTTPError("GET unrecoverable",
4217                                            410, "Gone", "NoSharesError",
4218                                            self.GET, self.fileurls["0shares"]))
4219         def _check_zero_shares(body):
4220             self.failIf("<html>" in body, body)
4221             body = " ".join(body.strip().split())
4222             exp = ("NoSharesError: no shares could be found. "
4223                    "Zero shares usually indicates a corrupt URI, or that "
4224                    "no servers were connected, but it might also indicate "
4225                    "severe corruption. You should perform a filecheck on "
4226                    "this object to learn more. The full error message is: "
4227                    "no shares (need 3). Last failure: None")
4228             self.failUnlessReallyEqual(exp, body)
4229         d.addCallback(_check_zero_shares)
4230
4231
4232         d.addCallback(lambda ignored:
4233                       self.shouldHTTPError("GET 1share",
4234                                            410, "Gone", "NotEnoughSharesError",
4235                                            self.GET, self.fileurls["1share"]))
4236         def _check_one_share(body):
4237             self.failIf("<html>" in body, body)
4238             body = " ".join(body.strip().split())
4239             msg = ("NotEnoughSharesError: This indicates that some "
4240                    "servers were unavailable, or that shares have been "
4241                    "lost to server departure, hard drive failure, or disk "
4242                    "corruption. You should perform a filecheck on "
4243                    "this object to learn more. The full error message is:"
4244                    " ran out of shares: %d complete, %d pending, 0 overdue,"
4245                    " 0 unused, need 3. Last failure: None")
4246             msg1 = msg % (1, 0)
4247             msg2 = msg % (0, 1)
4248             self.failUnless(body == msg1 or body == msg2, body)
4249         d.addCallback(_check_one_share)
4250
4251         d.addCallback(lambda ignored:
4252                       self.shouldHTTPError("GET imaginary",
4253                                            404, "Not Found", None,
4254                                            self.GET, self.fileurls["imaginary"]))
4255         def _missing_child(body):
4256             self.failUnless("No such child: imaginary" in body, body)
4257         d.addCallback(_missing_child)
4258
4259         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4260         def _check_0shares_dir_html(body):
4261             self.failUnless("<html>" in body, body)
4262             # we should see the regular page, but without the child table or
4263             # the dirops forms
4264             body = " ".join(body.strip().split())
4265             self.failUnlessIn('href="?t=info">More info on this directory',
4266                               body)
4267             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4268                    "could not be retrieved, because there were insufficient "
4269                    "good shares. This might indicate that no servers were "
4270                    "connected, insufficient servers were connected, the URI "
4271                    "was corrupt, or that shares have been lost due to server "
4272                    "departure, hard drive failure, or disk corruption. You "
4273                    "should perform a filecheck on this object to learn more.")
4274             self.failUnlessIn(exp, body)
4275             self.failUnlessIn("No upload forms: directory is unreadable", body)
4276         d.addCallback(_check_0shares_dir_html)
4277
4278         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4279         def _check_1shares_dir_html(body):
4280             # at some point, we'll split UnrecoverableFileError into 0-shares
4281             # and some-shares like we did for immutable files (since there
4282             # are different sorts of advice to offer in each case). For now,
4283             # they present the same way.
4284             self.failUnless("<html>" in body, body)
4285             body = " ".join(body.strip().split())
4286             self.failUnlessIn('href="?t=info">More info on this directory',
4287                               body)
4288             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4289                    "could not be retrieved, because there were insufficient "
4290                    "good shares. This might indicate that no servers were "
4291                    "connected, insufficient servers were connected, the URI "
4292                    "was corrupt, or that shares have been lost due to server "
4293                    "departure, hard drive failure, or disk corruption. You "
4294                    "should perform a filecheck on this object to learn more.")
4295             self.failUnlessIn(exp, body)
4296             self.failUnlessIn("No upload forms: directory is unreadable", body)
4297         d.addCallback(_check_1shares_dir_html)
4298
4299         d.addCallback(lambda ignored:
4300                       self.shouldHTTPError("GET dir-0share-json",
4301                                            410, "Gone", "UnrecoverableFileError",
4302                                            self.GET,
4303                                            self.fileurls["dir-0share-json"]))
4304         def _check_unrecoverable_file(body):
4305             self.failIf("<html>" in body, body)
4306             body = " ".join(body.strip().split())
4307             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4308                    "could not be retrieved, because there were insufficient "
4309                    "good shares. This might indicate that no servers were "
4310                    "connected, insufficient servers were connected, the URI "
4311                    "was corrupt, or that shares have been lost due to server "
4312                    "departure, hard drive failure, or disk corruption. You "
4313                    "should perform a filecheck on this object to learn more.")
4314             self.failUnlessReallyEqual(exp, body)
4315         d.addCallback(_check_unrecoverable_file)
4316
4317         d.addCallback(lambda ignored:
4318                       self.shouldHTTPError("GET dir-1share-json",
4319                                            410, "Gone", "UnrecoverableFileError",
4320                                            self.GET,
4321                                            self.fileurls["dir-1share-json"]))
4322         d.addCallback(_check_unrecoverable_file)
4323
4324         d.addCallback(lambda ignored:
4325                       self.shouldHTTPError("GET imaginary",
4326                                            404, "Not Found", None,
4327                                            self.GET, self.fileurls["imaginary"]))
4328
4329         # attach a webapi child that throws a random error, to test how it
4330         # gets rendered.
4331         w = c0.getServiceNamed("webish")
4332         w.root.putChild("ERRORBOOM", ErrorBoom())
4333
4334         # "Accept: */*" :        should get a text/html stack trace
4335         # "Accept: text/plain" : should get a text/plain stack trace
4336         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4337         # no Accept header:      should get a text/html stack trace
4338
4339         d.addCallback(lambda ignored:
4340                       self.shouldHTTPError("GET errorboom_html",
4341                                            500, "Internal Server Error", None,
4342                                            self.GET, "ERRORBOOM",
4343                                            headers={"accept": ["*/*"]}))
4344         def _internal_error_html1(body):
4345             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4346         d.addCallback(_internal_error_html1)
4347
4348         d.addCallback(lambda ignored:
4349                       self.shouldHTTPError("GET errorboom_text",
4350                                            500, "Internal Server Error", None,
4351                                            self.GET, "ERRORBOOM",
4352                                            headers={"accept": ["text/plain"]}))
4353         def _internal_error_text2(body):
4354             self.failIf("<html>" in body, body)
4355             self.failUnless(body.startswith("Traceback "), body)
4356         d.addCallback(_internal_error_text2)
4357
4358         CLI_accepts = "text/plain, application/octet-stream"
4359         d.addCallback(lambda ignored:
4360                       self.shouldHTTPError("GET errorboom_text",
4361                                            500, "Internal Server Error", None,
4362                                            self.GET, "ERRORBOOM",
4363                                            headers={"accept": [CLI_accepts]}))
4364         def _internal_error_text3(body):
4365             self.failIf("<html>" in body, body)
4366             self.failUnless(body.startswith("Traceback "), body)
4367         d.addCallback(_internal_error_text3)
4368
4369         d.addCallback(lambda ignored:
4370                       self.shouldHTTPError("GET errorboom_text",
4371                                            500, "Internal Server Error", None,
4372                                            self.GET, "ERRORBOOM"))
4373         def _internal_error_html4(body):
4374             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4375         d.addCallback(_internal_error_html4)
4376
4377         def _flush_errors(res):
4378             # Trial: please ignore the CompletelyUnhandledError in the logs
4379             self.flushLoggedErrors(CompletelyUnhandledError)
4380             return res
4381         d.addBoth(_flush_errors)
4382
4383         return d
4384
4385 class CompletelyUnhandledError(Exception):
4386     pass
4387 class ErrorBoom(rend.Page):
4388     def beforeRender(self, ctx):
4389         raise CompletelyUnhandledError("whoops")