src/allmydata/test/test_web.py

   1
   2 import os.path, re, urllib, time
   3 import simplejson
   4 from StringIO import StringIO
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11 from nevow import rend
  12 from allmydata import interfaces, uri, webish, dirnode
  13 from allmydata.storage.shares import get_share_file
  14 from allmydata.storage_client import StorageFarmBroker
  15 from allmydata.immutable import upload
  16 from allmydata.immutable.downloader.status import DownloadStatus
  17 from allmydata.dirnode import DirectoryNode
  18 from allmydata.nodemaker import NodeMaker
  19 from allmydata.unknown import UnknownNode
  20 from allmydata.web import status, common
  21 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  22 from allmydata.util import fileutil, base32, hashutil
  23 from allmydata.util.consumer import download_to_data
  24 from allmydata.util.netstring import split_netstring
  25 from allmydata.util.encodingutil import to_str
  26 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  27      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  28 from allmydata.interfaces import IMutableFileNode
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     def _create_lit(self, cap):
  52         return FakeCHKFileNode(cap)
  53     def _create_immutable(self, cap):
  54         return FakeCHKFileNode(cap)
  55     def _create_mutable(self, cap):
  56         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  57     def create_mutable_file(self, contents="", keysize=None):
  58         n = FakeMutableFileNode(None, None, None, None)
  59         return n.create(contents)
  60
  61 class FakeUploader(service.Service):
  62     name = "uploader"
  63     def upload(self, uploadable, history=None):
  64         d = uploadable.get_size()
  65         d.addCallback(lambda size: uploadable.read(size))
  66         def _got_data(datav):
  67             data = "".join(datav)
  68             n = create_chk_filenode(data)
  69             results = upload.UploadResults()
  70             results.uri = n.get_uri()
  71             return results
  72         d.addCallback(_got_data)
  73         return d
  74     def get_helper_info(self):
  75         return (None, False)
  76
  77 class FakeIServer:
  78     def __init__(self, binaryserverid):
  79         self.binaryserverid = binaryserverid
  80     def get_name(self): return "short"
  81     def get_longname(self): return "long"
  82     def get_serverid(self): return self.binaryserverid
  83
  84 def build_one_ds():
  85     ds = DownloadStatus("storage_index", 1234)
  86     now = time.time()
  87
  88     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  89     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  90     storage_index = hashutil.storage_index_hash("SI")
  91     e0 = ds.add_segment_request(0, now)
  92     e0.activate(now+0.5)
  93     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
  94     e1 = ds.add_segment_request(1, now+2)
  95     e1.error(now+3)
  96     # two outstanding requests
  97     e2 = ds.add_segment_request(2, now+4)
  98     e3 = ds.add_segment_request(3, now+5)
  99     del e2,e3 # hush pyflakes
 100
 101     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 102     e = ds.add_segment_request(4, now)
 103     e.activate(now)
 104     e.deliver(now, 0, 140, 0.5)
 105
 106     e = ds.add_dyhb_request(serverA, now)
 107     e.finished([1,2], now+1)
 108     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 109
 110     e = ds.add_read_event(0, 120, now)
 111     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 112     e.finished(now+1)
 113     e = ds.add_read_event(120, 30, now+2) # left unfinished
 114
 115     e = ds.add_block_request(serverA, 1, 100, 20, now)
 116     e.finished(20, now+1)
 117     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 118
 119     # make sure that add_read_event() can come first too
 120     ds1 = DownloadStatus(storage_index, 1234)
 121     e = ds1.add_read_event(0, 120, now)
 122     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 123     e.finished(now+1)
 124
 125     return ds
 126
 127 class FakeHistory:
 128     _all_upload_status = [upload.UploadStatus()]
 129     _all_download_status = [build_one_ds()]
 130     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 131     _all_publish_statuses = [publish.PublishStatus()]
 132     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 133
 134     def list_all_upload_statuses(self):
 135         return self._all_upload_status
 136     def list_all_download_statuses(self):
 137         return self._all_download_status
 138     def list_all_mapupdate_statuses(self):
 139         return self._all_mapupdate_statuses
 140     def list_all_publish_statuses(self):
 141         return self._all_publish_statuses
 142     def list_all_retrieve_statuses(self):
 143         return self._all_retrieve_statuses
 144     def list_all_helper_statuses(self):
 145         return []
 146
 147 class FakeClient(Client):
 148     def __init__(self):
 149         # don't upcall to Client.__init__, since we only want to initialize a
 150         # minimal subset
 151         service.MultiService.__init__(self)
 152         self.nodeid = "fake_nodeid"
 153         self.nickname = "fake_nickname"
 154         self.introducer_furl = "None"
 155         self.stats_provider = FakeStatsProvider()
 156         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 157         self.helper = None
 158         self.convergence = "some random string"
 159         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 160         self.introducer_client = None
 161         self.history = FakeHistory()
 162         self.uploader = FakeUploader()
 163         self.uploader.setServiceParent(self)
 164         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 165                                        self.uploader, None,
 166                                        None, None)
 167
 168     def startService(self):
 169         return service.MultiService.startService(self)
 170     def stopService(self):
 171         return service.MultiService.stopService(self)
 172
 173     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 174
 175 class WebMixin(object):
 176     def setUp(self):
 177         self.s = FakeClient()
 178         self.s.startService()
 179         self.staticdir = self.mktemp()
 180         self.clock = Clock()
 181         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 182                                       clock=self.clock)
 183         self.ws.setServiceParent(self.s)
 184         self.webish_port = self.ws.getPortnum()
 185         self.webish_url = self.ws.getURL()
 186         assert self.webish_url.endswith("/")
 187         self.webish_url = self.webish_url[:-1] # these tests add their own /
 188
 189         l = [ self.s.create_dirnode() for x in range(6) ]
 190         d = defer.DeferredList(l)
 191         def _then(res):
 192             self.public_root = res[0][1]
 193             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 194             self.public_url = "/uri/" + self.public_root.get_uri()
 195             self.private_root = res[1][1]
 196
 197             foo = res[2][1]
 198             self._foo_node = foo
 199             self._foo_uri = foo.get_uri()
 200             self._foo_readonly_uri = foo.get_readonly_uri()
 201             self._foo_verifycap = foo.get_verify_cap().to_string()
 202             # NOTE: we ignore the deferred on all set_uri() calls, because we
 203             # know the fake nodes do these synchronously
 204             self.public_root.set_uri(u"foo", foo.get_uri(),
 205                                      foo.get_readonly_uri())
 206
 207             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 208             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 209             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 210
 211             foo.set_uri(u"empty", res[3][1].get_uri(),
 212                         res[3][1].get_readonly_uri())
 213             sub_uri = res[4][1].get_uri()
 214             self._sub_uri = sub_uri
 215             foo.set_uri(u"sub", sub_uri, sub_uri)
 216             sub = self.s.create_node_from_uri(sub_uri)
 217
 218             _ign, n, blocking_uri = self.makefile(1)
 219             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 220
 221             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 222             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 223             # still think of it as an umlaut
 224             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 225
 226             _ign, n, baz_file = self.makefile(2)
 227             self._baz_file_uri = baz_file
 228             sub.set_uri(u"baz.txt", baz_file, baz_file)
 229
 230             _ign, n, self._bad_file_uri = self.makefile(3)
 231             # this uri should not be downloadable
 232             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 233
 234             rodir = res[5][1]
 235             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 236                                      rodir.get_readonly_uri())
 237             rodir.set_uri(u"nor", baz_file, baz_file)
 238
 239             # public/
 240             # public/foo/
 241             # public/foo/bar.txt
 242             # public/foo/blockingfile
 243             # public/foo/empty/
 244             # public/foo/sub/
 245             # public/foo/sub/baz.txt
 246             # public/reedownlee/
 247             # public/reedownlee/nor
 248             self.NEWFILE_CONTENTS = "newfile contents\n"
 249
 250             return foo.get_metadata_for(u"bar.txt")
 251         d.addCallback(_then)
 252         def _got_metadata(metadata):
 253             self._bar_txt_metadata = metadata
 254         d.addCallback(_got_metadata)
 255         return d
 256
 257     def makefile(self, number):
 258         contents = "contents of file %s\n" % number
 259         n = create_chk_filenode(contents)
 260         return contents, n, n.get_uri()
 261
 262     def tearDown(self):
 263         return self.s.stopService()
 264
 265     def failUnlessIsBarDotTxt(self, res):
 266         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 267
 268     def failUnlessIsBarJSON(self, res):
 269         data = simplejson.loads(res)
 270         self.failUnless(isinstance(data, list))
 271         self.failUnlessEqual(data[0], "filenode")
 272         self.failUnless(isinstance(data[1], dict))
 273         self.failIf(data[1]["mutable"])
 274         self.failIf("rw_uri" in data[1]) # immutable
 275         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 276         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 277         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 278
 279     def failUnlessIsFooJSON(self, res):
 280         data = simplejson.loads(res)
 281         self.failUnless(isinstance(data, list))
 282         self.failUnlessEqual(data[0], "dirnode", res)
 283         self.failUnless(isinstance(data[1], dict))
 284         self.failUnless(data[1]["mutable"])
 285         self.failUnless("rw_uri" in data[1]) # mutable
 286         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 287         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 288         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 289
 290         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 291         self.failUnlessEqual(kidnames,
 292                              [u"bar.txt", u"blockingfile", u"empty",
 293                               u"n\u00fc.txt", u"sub"])
 294         kids = dict( [(unicode(name),value)
 295                       for (name,value)
 296                       in data[1]["children"].iteritems()] )
 297         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 298         self.failUnlessIn("metadata", kids[u"sub"][1])
 299         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 300         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 301         self.failUnlessIn("linkcrtime", tahoe_md)
 302         self.failUnlessIn("linkmotime", tahoe_md)
 303         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 304         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 305         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 306         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 307                                    self._bar_txt_verifycap)
 308         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 309         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 310         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 311                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 312         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 313                                    self._bar_txt_uri)
 314
 315     def GET(self, urlpath, followRedirect=False, return_response=False,
 316             **kwargs):
 317         # if return_response=True, this fires with (data, statuscode,
 318         # respheaders) instead of just data.
 319         assert not isinstance(urlpath, unicode)
 320         url = self.webish_url + urlpath
 321         factory = HTTPClientGETFactory(url, method="GET",
 322                                        followRedirect=followRedirect, **kwargs)
 323         reactor.connectTCP("localhost", self.webish_port, factory)
 324         d = factory.deferred
 325         def _got_data(data):
 326             return (data, factory.status, factory.response_headers)
 327         if return_response:
 328             d.addCallback(_got_data)
 329         return factory.deferred
 330
 331     def HEAD(self, urlpath, return_response=False, **kwargs):
 332         # this requires some surgery, because twisted.web.client doesn't want
 333         # to give us back the response headers.
 334         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 335         reactor.connectTCP("localhost", self.webish_port, factory)
 336         d = factory.deferred
 337         def _got_data(data):
 338             return (data, factory.status, factory.response_headers)
 339         if return_response:
 340             d.addCallback(_got_data)
 341         return factory.deferred
 342
 343     def PUT(self, urlpath, data, **kwargs):
 344         url = self.webish_url + urlpath
 345         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 346
 347     def DELETE(self, urlpath):
 348         url = self.webish_url + urlpath
 349         return client.getPage(url, method="DELETE")
 350
 351     def POST(self, urlpath, followRedirect=False, **fields):
 352         sepbase = "boogabooga"
 353         sep = "--" + sepbase
 354         form = []
 355         form.append(sep)
 356         form.append('Content-Disposition: form-data; name="_charset"')
 357         form.append('')
 358         form.append('UTF-8')
 359         form.append(sep)
 360         for name, value in fields.iteritems():
 361             if isinstance(value, tuple):
 362                 filename, value = value
 363                 form.append('Content-Disposition: form-data; name="%s"; '
 364                             'filename="%s"' % (name, filename.encode("utf-8")))
 365             else:
 366                 form.append('Content-Disposition: form-data; name="%s"' % name)
 367             form.append('')
 368             if isinstance(value, unicode):
 369                 value = value.encode("utf-8")
 370             else:
 371                 value = str(value)
 372             assert isinstance(value, str)
 373             form.append(value)
 374             form.append(sep)
 375         form[-1] += "--"
 376         body = ""
 377         headers = {}
 378         if fields:
 379             body = "\r\n".join(form) + "\r\n"
 380             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 381         return self.POST2(urlpath, body, headers, followRedirect)
 382
 383     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 384         url = self.webish_url + urlpath
 385         return client.getPage(url, method="POST", postdata=body,
 386                               headers=headers, followRedirect=followRedirect)
 387
 388     def shouldFail(self, res, expected_failure, which,
 389                    substring=None, response_substring=None):
 390         if isinstance(res, failure.Failure):
 391             res.trap(expected_failure)
 392             if substring:
 393                 self.failUnless(substring in str(res),
 394                                 "substring '%s' not in '%s'"
 395                                 % (substring, str(res)))
 396             if response_substring:
 397                 self.failUnless(response_substring in res.value.response,
 398                                 "response substring '%s' not in '%s'"
 399                                 % (response_substring, res.value.response))
 400         else:
 401             self.fail("%s was supposed to raise %s, not get '%s'" %
 402                       (which, expected_failure, res))
 403
 404     def shouldFail2(self, expected_failure, which, substring,
 405                     response_substring,
 406                     callable, *args, **kwargs):
 407         assert substring is None or isinstance(substring, str)
 408         assert response_substring is None or isinstance(response_substring, str)
 409         d = defer.maybeDeferred(callable, *args, **kwargs)
 410         def done(res):
 411             if isinstance(res, failure.Failure):
 412                 res.trap(expected_failure)
 413                 if substring:
 414                     self.failUnless(substring in str(res),
 415                                     "%s: substring '%s' not in '%s'"
 416                                     % (which, substring, str(res)))
 417                 if response_substring:
 418                     self.failUnless(response_substring in res.value.response,
 419                                     "%s: response substring '%s' not in '%s'"
 420                                     % (which,
 421                                        response_substring, res.value.response))
 422             else:
 423                 self.fail("%s was supposed to raise %s, not get '%s'" %
 424                           (which, expected_failure, res))
 425         d.addBoth(done)
 426         return d
 427
 428     def should404(self, res, which):
 429         if isinstance(res, failure.Failure):
 430             res.trap(error.Error)
 431             self.failUnlessReallyEqual(res.value.status, "404")
 432         else:
 433             self.fail("%s was supposed to Error(404), not get '%s'" %
 434                       (which, res))
 435
 436     def should302(self, res, which):
 437         if isinstance(res, failure.Failure):
 438             res.trap(error.Error)
 439             self.failUnlessReallyEqual(res.value.status, "302")
 440         else:
 441             self.fail("%s was supposed to Error(302), not get '%s'" %
 442                         (which, res))
 443
 444
 445 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 446     def test_create(self):
 447         pass
 448
 449     def test_welcome(self):
 450         d = self.GET("/")
 451         def _check(res):
 452             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 453
 454             self.s.basedir = 'web/test_welcome'
 455             fileutil.make_dirs("web/test_welcome")
 456             fileutil.make_dirs("web/test_welcome/private")
 457             return self.GET("/")
 458         d.addCallback(_check)
 459         return d
 460
 461     def test_provisioning(self):
 462         d = self.GET("/provisioning/")
 463         def _check(res):
 464             self.failUnless('Provisioning Tool' in res)
 465             fields = {'filled': True,
 466                       "num_users": int(50e3),
 467                       "files_per_user": 1000,
 468                       "space_per_user": int(1e9),
 469                       "sharing_ratio": 1.0,
 470                       "encoding_parameters": "3-of-10-5",
 471                       "num_servers": 30,
 472                       "ownership_mode": "A",
 473                       "download_rate": 100,
 474                       "upload_rate": 10,
 475                       "delete_rate": 10,
 476                       "lease_timer": 7,
 477                       }
 478             return self.POST("/provisioning/", **fields)
 479
 480         d.addCallback(_check)
 481         def _check2(res):
 482             self.failUnless('Provisioning Tool' in res)
 483             self.failUnless("Share space consumed: 167.01TB" in res)
 484
 485             fields = {'filled': True,
 486                       "num_users": int(50e6),
 487                       "files_per_user": 1000,
 488                       "space_per_user": int(5e9),
 489                       "sharing_ratio": 1.0,
 490                       "encoding_parameters": "25-of-100-50",
 491                       "num_servers": 30000,
 492                       "ownership_mode": "E",
 493                       "drive_failure_model": "U",
 494                       "drive_size": 1000,
 495                       "download_rate": 1000,
 496                       "upload_rate": 100,
 497                       "delete_rate": 100,
 498                       "lease_timer": 7,
 499                       }
 500             return self.POST("/provisioning/", **fields)
 501         d.addCallback(_check2)
 502         def _check3(res):
 503             self.failUnless("Share space consumed: huge!" in res)
 504             fields = {'filled': True}
 505             return self.POST("/provisioning/", **fields)
 506         d.addCallback(_check3)
 507         def _check4(res):
 508             self.failUnless("Share space consumed:" in res)
 509         d.addCallback(_check4)
 510         return d
 511
 512     def test_reliability_tool(self):
 513         try:
 514             from allmydata import reliability
 515             _hush_pyflakes = reliability
 516             del _hush_pyflakes
 517         except:
 518             raise unittest.SkipTest("reliability tool requires NumPy")
 519
 520         d = self.GET("/reliability/")
 521         def _check(res):
 522             self.failUnless('Reliability Tool' in res)
 523             fields = {'drive_lifetime': "8Y",
 524                       "k": "3",
 525                       "R": "7",
 526                       "N": "10",
 527                       "delta": "100000",
 528                       "check_period": "1M",
 529                       "report_period": "3M",
 530                       "report_span": "5Y",
 531                       }
 532             return self.POST("/reliability/", **fields)
 533
 534         d.addCallback(_check)
 535         def _check2(res):
 536             self.failUnless('Reliability Tool' in res)
 537             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 538             self.failUnless(re.search(r, res), res)
 539         d.addCallback(_check2)
 540         return d
 541
 542     def test_status(self):
 543         h = self.s.get_history()
 544         dl_num = h.list_all_download_statuses()[0].get_counter()
 545         ul_num = h.list_all_upload_statuses()[0].get_counter()
 546         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 547         pub_num = h.list_all_publish_statuses()[0].get_counter()
 548         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 549         d = self.GET("/status", followRedirect=True)
 550         def _check(res):
 551             self.failUnless('Upload and Download Status' in res, res)
 552             self.failUnless('"down-%d"' % dl_num in res, res)
 553             self.failUnless('"up-%d"' % ul_num in res, res)
 554             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 555             self.failUnless('"publish-%d"' % pub_num in res, res)
 556             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 557         d.addCallback(_check)
 558         d.addCallback(lambda res: self.GET("/status/?t=json"))
 559         def _check_json(res):
 560             data = simplejson.loads(res)
 561             self.failUnless(isinstance(data, dict))
 562             #active = data["active"]
 563             # TODO: test more. We need a way to fake an active operation
 564             # here.
 565         d.addCallback(_check_json)
 566
 567         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 568         def _check_dl(res):
 569             self.failUnless("File Download Status" in res, res)
 570         d.addCallback(_check_dl)
 571         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 572         def _check_dl_json(res):
 573             data = simplejson.loads(res)
 574             self.failUnless(isinstance(data, dict))
 575             self.failUnless("read" in data)
 576             self.failUnlessEqual(data["read"][0]["length"], 120)
 577             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 578             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 579             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 580             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 581             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 582             # serverids[] keys are strings, since that's what JSON does, but
 583             # we'd really like them to be ints
 584             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 585             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 586             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 587             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 588             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 589             self.failUnless("dyhb" in data)
 590         d.addCallback(_check_dl_json)
 591         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 592         def _check_ul(res):
 593             self.failUnless("File Upload Status" in res, res)
 594         d.addCallback(_check_ul)
 595         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 596         def _check_mapupdate(res):
 597             self.failUnless("Mutable File Servermap Update Status" in res, res)
 598         d.addCallback(_check_mapupdate)
 599         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 600         def _check_publish(res):
 601             self.failUnless("Mutable File Publish Status" in res, res)
 602         d.addCallback(_check_publish)
 603         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 604         def _check_retrieve(res):
 605             self.failUnless("Mutable File Retrieve Status" in res, res)
 606         d.addCallback(_check_retrieve)
 607
 608         return d
 609
 610     def test_status_numbers(self):
 611         drrm = status.DownloadResultsRendererMixin()
 612         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 613         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 614         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 615         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 616         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 617         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 618         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 619         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 620         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 621
 622         urrm = status.UploadResultsRendererMixin()
 623         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 624         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 625         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 626         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 627         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 628         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 629         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 630         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 631         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 632
 633     def test_GET_FILEURL(self):
 634         d = self.GET(self.public_url + "/foo/bar.txt")
 635         d.addCallback(self.failUnlessIsBarDotTxt)
 636         return d
 637
 638     def test_GET_FILEURL_range(self):
 639         headers = {"range": "bytes=1-10"}
 640         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 641                      return_response=True)
 642         def _got((res, status, headers)):
 643             self.failUnlessReallyEqual(int(status), 206)
 644             self.failUnless(headers.has_key("content-range"))
 645             self.failUnlessReallyEqual(headers["content-range"][0],
 646                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 647             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 648         d.addCallback(_got)
 649         return d
 650
 651     def test_GET_FILEURL_partial_range(self):
 652         headers = {"range": "bytes=5-"}
 653         length  = len(self.BAR_CONTENTS)
 654         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 655                      return_response=True)
 656         def _got((res, status, headers)):
 657             self.failUnlessReallyEqual(int(status), 206)
 658             self.failUnless(headers.has_key("content-range"))
 659             self.failUnlessReallyEqual(headers["content-range"][0],
 660                                        "bytes 5-%d/%d" % (length-1, length))
 661             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 662         d.addCallback(_got)
 663         return d
 664
 665     def test_GET_FILEURL_partial_end_range(self):
 666         headers = {"range": "bytes=-5"}
 667         length  = len(self.BAR_CONTENTS)
 668         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 669                      return_response=True)
 670         def _got((res, status, headers)):
 671             self.failUnlessReallyEqual(int(status), 206)
 672             self.failUnless(headers.has_key("content-range"))
 673             self.failUnlessReallyEqual(headers["content-range"][0],
 674                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 675             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 676         d.addCallback(_got)
 677         return d
 678
 679     def test_GET_FILEURL_partial_range_overrun(self):
 680         headers = {"range": "bytes=100-200"}
 681         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 682                              "416 Requested Range not satisfiable",
 683                              "First beyond end of file",
 684                              self.GET, self.public_url + "/foo/bar.txt",
 685                              headers=headers)
 686         return d
 687
 688     def test_HEAD_FILEURL_range(self):
 689         headers = {"range": "bytes=1-10"}
 690         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 691                      return_response=True)
 692         def _got((res, status, headers)):
 693             self.failUnlessReallyEqual(res, "")
 694             self.failUnlessReallyEqual(int(status), 206)
 695             self.failUnless(headers.has_key("content-range"))
 696             self.failUnlessReallyEqual(headers["content-range"][0],
 697                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 698         d.addCallback(_got)
 699         return d
 700
 701     def test_HEAD_FILEURL_partial_range(self):
 702         headers = {"range": "bytes=5-"}
 703         length  = len(self.BAR_CONTENTS)
 704         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 705                      return_response=True)
 706         def _got((res, status, headers)):
 707             self.failUnlessReallyEqual(int(status), 206)
 708             self.failUnless(headers.has_key("content-range"))
 709             self.failUnlessReallyEqual(headers["content-range"][0],
 710                                        "bytes 5-%d/%d" % (length-1, length))
 711         d.addCallback(_got)
 712         return d
 713
 714     def test_HEAD_FILEURL_partial_end_range(self):
 715         headers = {"range": "bytes=-5"}
 716         length  = len(self.BAR_CONTENTS)
 717         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 718                      return_response=True)
 719         def _got((res, status, headers)):
 720             self.failUnlessReallyEqual(int(status), 206)
 721             self.failUnless(headers.has_key("content-range"))
 722             self.failUnlessReallyEqual(headers["content-range"][0],
 723                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 724         d.addCallback(_got)
 725         return d
 726
 727     def test_HEAD_FILEURL_partial_range_overrun(self):
 728         headers = {"range": "bytes=100-200"}
 729         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 730                              "416 Requested Range not satisfiable",
 731                              "",
 732                              self.HEAD, self.public_url + "/foo/bar.txt",
 733                              headers=headers)
 734         return d
 735
 736     def test_GET_FILEURL_range_bad(self):
 737         headers = {"range": "BOGUS=fizbop-quarnak"}
 738         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 739                      return_response=True)
 740         def _got((res, status, headers)):
 741             self.failUnlessReallyEqual(int(status), 200)
 742             self.failUnless(not headers.has_key("content-range"))
 743             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_HEAD_FILEURL(self):
 748         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 749         def _got((res, status, headers)):
 750             self.failUnlessReallyEqual(res, "")
 751             self.failUnlessReallyEqual(headers["content-length"][0],
 752                                        str(len(self.BAR_CONTENTS)))
 753             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 754         d.addCallback(_got)
 755         return d
 756
 757     def test_GET_FILEURL_named(self):
 758         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 759         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 760         d = self.GET(base + "/@@name=/blah.txt")
 761         d.addCallback(self.failUnlessIsBarDotTxt)
 762         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 763         d.addCallback(self.failUnlessIsBarDotTxt)
 764         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 765         d.addCallback(self.failUnlessIsBarDotTxt)
 766         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 767         d.addCallback(self.failUnlessIsBarDotTxt)
 768         save_url = base + "?save=true&filename=blah.txt"
 769         d.addCallback(lambda res: self.GET(save_url))
 770         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 771         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 772         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 773         u_url = base + "?save=true&filename=" + u_fn_e
 774         d.addCallback(lambda res: self.GET(u_url))
 775         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 776         return d
 777
 778     def test_PUT_FILEURL_named_bad(self):
 779         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 780         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 781                              "400 Bad Request",
 782                              "/file can only be used with GET or HEAD",
 783                              self.PUT, base + "/@@name=/blah.txt", "")
 784         return d
 785
 786     def test_GET_DIRURL_named_bad(self):
 787         base = "/file/%s" % urllib.quote(self._foo_uri)
 788         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 789                              "400 Bad Request",
 790                              "is not a file-cap",
 791                              self.GET, base + "/@@name=/blah.txt")
 792         return d
 793
 794     def test_GET_slash_file_bad(self):
 795         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 796                              "404 Not Found",
 797                              "/file must be followed by a file-cap and a name",
 798                              self.GET, "/file")
 799         return d
 800
 801     def test_GET_unhandled_URI_named(self):
 802         contents, n, newuri = self.makefile(12)
 803         verifier_cap = n.get_verify_cap().to_string()
 804         base = "/file/%s" % urllib.quote(verifier_cap)
 805         # client.create_node_from_uri() can't handle verify-caps
 806         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 807                              "400 Bad Request", "is not a file-cap",
 808                              self.GET, base)
 809         return d
 810
 811     def test_GET_unhandled_URI(self):
 812         contents, n, newuri = self.makefile(12)
 813         verifier_cap = n.get_verify_cap().to_string()
 814         base = "/uri/%s" % urllib.quote(verifier_cap)
 815         # client.create_node_from_uri() can't handle verify-caps
 816         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 817                              "400 Bad Request",
 818                              "GET unknown URI type: can only do t=info",
 819                              self.GET, base)
 820         return d
 821
 822     def test_GET_FILE_URI(self):
 823         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 824         d = self.GET(base)
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         return d
 827
 828     def test_GET_FILE_URI_badchild(self):
 829         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 830         errmsg = "Files have no children, certainly not named 'boguschild'"
 831         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 832                              "400 Bad Request", errmsg,
 833                              self.GET, base)
 834         return d
 835
 836     def test_PUT_FILE_URI_badchild(self):
 837         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 838         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 839         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 840                              "400 Bad Request", errmsg,
 841                              self.PUT, base, "")
 842         return d
 843
 844     # TODO: version of this with a Unicode filename
 845     def test_GET_FILEURL_save(self):
 846         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 847                      return_response=True)
 848         def _got((res, statuscode, headers)):
 849             content_disposition = headers["content-disposition"][0]
 850             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 851             self.failUnlessIsBarDotTxt(res)
 852         d.addCallback(_got)
 853         return d
 854
 855     def test_GET_FILEURL_missing(self):
 856         d = self.GET(self.public_url + "/foo/missing")
 857         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 858         return d
 859
 860     def test_PUT_overwrite_only_files(self):
 861         # create a directory, put a file in that directory.
 862         contents, n, filecap = self.makefile(8)
 863         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 864         d.addCallback(lambda res:
 865             self.PUT(self.public_url + "/foo/dir/file1.txt",
 866                      self.NEWFILE_CONTENTS))
 867         # try to overwrite the file with replace=only-files
 868         # (this should work)
 869         d.addCallback(lambda res:
 870             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 871                      filecap))
 872         d.addCallback(lambda res:
 873             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 874                  "There was already a child by that name, and you asked me "
 875                  "to not replace it",
 876                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 877                  filecap))
 878         return d
 879
 880     def test_PUT_NEWFILEURL(self):
 881         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 882         # TODO: we lose the response code, so we can't check this
 883         #self.failUnlessReallyEqual(responsecode, 201)
 884         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 885         d.addCallback(lambda res:
 886                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 887                                                       self.NEWFILE_CONTENTS))
 888         return d
 889
 890     def test_PUT_NEWFILEURL_not_mutable(self):
 891         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 892                      self.NEWFILE_CONTENTS)
 893         # TODO: we lose the response code, so we can't check this
 894         #self.failUnlessReallyEqual(responsecode, 201)
 895         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 896         d.addCallback(lambda res:
 897                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 898                                                       self.NEWFILE_CONTENTS))
 899         return d
 900
 901     def test_PUT_NEWFILEURL_range_bad(self):
 902         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 903         target = self.public_url + "/foo/new.txt"
 904         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 905                              "501 Not Implemented",
 906                              "Content-Range in PUT not yet supported",
 907                              # (and certainly not for immutable files)
 908                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 909                              headers=headers)
 910         d.addCallback(lambda res:
 911                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 912         return d
 913
 914     def test_PUT_NEWFILEURL_mutable(self):
 915         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 916                      self.NEWFILE_CONTENTS)
 917         # TODO: we lose the response code, so we can't check this
 918         #self.failUnlessReallyEqual(responsecode, 201)
 919         def _check_uri(res):
 920             u = uri.from_string_mutable_filenode(res)
 921             self.failUnless(u.is_mutable())
 922             self.failIf(u.is_readonly())
 923             return res
 924         d.addCallback(_check_uri)
 925         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 926         d.addCallback(lambda res:
 927                       self.failUnlessMutableChildContentsAre(self._foo_node,
 928                                                              u"new.txt",
 929                                                              self.NEWFILE_CONTENTS))
 930         return d
 931
 932     def test_PUT_NEWFILEURL_mutable_toobig(self):
 933         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 934                              "413 Request Entity Too Large",
 935                              "SDMF is limited to one segment, and 10001 > 10000",
 936                              self.PUT,
 937                              self.public_url + "/foo/new.txt?mutable=true",
 938                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 939         return d
 940
 941     def test_PUT_NEWFILEURL_replace(self):
 942         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 943         # TODO: we lose the response code, so we can't check this
 944         #self.failUnlessReallyEqual(responsecode, 200)
 945         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 946         d.addCallback(lambda res:
 947                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 948                                                       self.NEWFILE_CONTENTS))
 949         return d
 950
 951     def test_PUT_NEWFILEURL_bad_t(self):
 952         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 953                              "PUT to a file: bad t=bogus",
 954                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 955                              "contents")
 956         return d
 957
 958     def test_PUT_NEWFILEURL_no_replace(self):
 959         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 960                      self.NEWFILE_CONTENTS)
 961         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 962                   "409 Conflict",
 963                   "There was already a child by that name, and you asked me "
 964                   "to not replace it")
 965         return d
 966
 967     def test_PUT_NEWFILEURL_mkdirs(self):
 968         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 969         fn = self._foo_node
 970         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 971         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 972         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 973         d.addCallback(lambda res:
 974                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 975                                                       self.NEWFILE_CONTENTS))
 976         return d
 977
 978     def test_PUT_NEWFILEURL_blocked(self):
 979         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 980                      self.NEWFILE_CONTENTS)
 981         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 982                   "409 Conflict",
 983                   "Unable to create directory 'blockingfile': a file was in the way")
 984         return d
 985
 986     def test_PUT_NEWFILEURL_emptyname(self):
 987         # an empty pathname component (i.e. a double-slash) is disallowed
 988         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 989                              "400 Bad Request",
 990                              "The webapi does not allow empty pathname components",
 991                              self.PUT, self.public_url + "/foo//new.txt", "")
 992         return d
 993
 994     def test_DELETE_FILEURL(self):
 995         d = self.DELETE(self.public_url + "/foo/bar.txt")
 996         d.addCallback(lambda res:
 997                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 998         return d
 999
1000     def test_DELETE_FILEURL_missing(self):
1001         d = self.DELETE(self.public_url + "/foo/missing")
1002         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1003         return d
1004
1005     def test_DELETE_FILEURL_missing2(self):
1006         d = self.DELETE(self.public_url + "/missing/missing")
1007         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1008         return d
1009
1010     def failUnlessHasBarDotTxtMetadata(self, res):
1011         data = simplejson.loads(res)
1012         self.failUnless(isinstance(data, list))
1013         self.failUnlessIn("metadata", data[1])
1014         self.failUnlessIn("tahoe", data[1]["metadata"])
1015         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1016         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1017         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1018                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1019
1020     def test_GET_FILEURL_json(self):
1021         # twisted.web.http.parse_qs ignores any query args without an '=', so
1022         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1023         # instead. This may make it tricky to emulate the S3 interface
1024         # completely.
1025         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1026         def _check1(data):
1027             self.failUnlessIsBarJSON(data)
1028             self.failUnlessHasBarDotTxtMetadata(data)
1029             return
1030         d.addCallback(_check1)
1031         return d
1032
1033     def test_GET_FILEURL_json_missing(self):
1034         d = self.GET(self.public_url + "/foo/missing?json")
1035         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1036         return d
1037
1038     def test_GET_FILEURL_uri(self):
1039         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1040         def _check(res):
1041             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1042         d.addCallback(_check)
1043         d.addCallback(lambda res:
1044                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1045         def _check2(res):
1046             # for now, for files, uris and readonly-uris are the same
1047             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1048         d.addCallback(_check2)
1049         return d
1050
1051     def test_GET_FILEURL_badtype(self):
1052         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1053                                  "bad t=bogus",
1054                                  self.GET,
1055                                  self.public_url + "/foo/bar.txt?t=bogus")
1056         return d
1057
1058     def test_CSS_FILE(self):
1059         d = self.GET("/tahoe_css", followRedirect=True)
1060         def _check(res):
1061             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1062             self.failUnless(CSS_STYLE.search(res), res)
1063         d.addCallback(_check)
1064         return d
1065
1066     def test_GET_FILEURL_uri_missing(self):
1067         d = self.GET(self.public_url + "/foo/missing?t=uri")
1068         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1069         return d
1070
1071     def test_GET_DIRECTORY_html_banner(self):
1072         d = self.GET(self.public_url + "/foo", followRedirect=True)
1073         def _check(res):
1074             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1075         d.addCallback(_check)
1076         return d
1077
1078     def test_GET_DIRURL(self):
1079         # the addSlash means we get a redirect here
1080         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1081         ROOT = "../../.."
1082         d = self.GET(self.public_url + "/foo", followRedirect=True)
1083         def _check(res):
1084             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1085                             in res, res)
1086             # the FILE reference points to a URI, but it should end in bar.txt
1087             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1088                        (ROOT, urllib.quote(self._bar_txt_uri)))
1089             get_bar = "".join([r'<td>FILE</td>',
1090                                r'\s+<td>',
1091                                r'<a href="%s">bar.txt</a>' % bar_url,
1092                                r'</td>',
1093                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1094                                ])
1095             self.failUnless(re.search(get_bar, res), res)
1096             for label in ['unlink', 'rename']:
1097                 for line in res.split("\n"):
1098                     # find the line that contains the relevant button for bar.txt
1099                     if ("form action" in line and
1100                         ('value="%s"' % (label,)) in line and
1101                         'value="bar.txt"' in line):
1102                         # the form target should use a relative URL
1103                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1104                         self.failUnlessIn('action="%s"' % foo_url, line)
1105                         # and the when_done= should too
1106                         #done_url = urllib.quote(???)
1107                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1108
1109                         # 'unlink' needs to use POST because it directly has a side effect
1110                         if label == 'unlink':
1111                             self.failUnlessIn('method="post"', line)
1112                         break
1113                 else:
1114                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1115
1116             # the DIR reference just points to a URI
1117             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1118             get_sub = ((r'<td>DIR</td>')
1119                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1120             self.failUnless(re.search(get_sub, res), res)
1121         d.addCallback(_check)
1122
1123         # look at a readonly directory
1124         d.addCallback(lambda res:
1125                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1126         def _check2(res):
1127             self.failUnless("(read-only)" in res, res)
1128             self.failIf("Upload a file" in res, res)
1129         d.addCallback(_check2)
1130
1131         # and at a directory that contains a readonly directory
1132         d.addCallback(lambda res:
1133                       self.GET(self.public_url, followRedirect=True))
1134         def _check3(res):
1135             self.failUnless(re.search('<td>DIR-RO</td>'
1136                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1137         d.addCallback(_check3)
1138
1139         # and an empty directory
1140         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1141         def _check4(res):
1142             self.failUnless("directory is empty" in res, res)
1143             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1144             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1145         d.addCallback(_check4)
1146
1147         # and at a literal directory
1148         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1149         d.addCallback(lambda res:
1150                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1151         def _check5(res):
1152             self.failUnless('(immutable)' in res, res)
1153             self.failUnless(re.search('<td>FILE</td>'
1154                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1155         d.addCallback(_check5)
1156         return d
1157
1158     def test_GET_DIRURL_badtype(self):
1159         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1160                                  400, "Bad Request",
1161                                  "bad t=bogus",
1162                                  self.GET,
1163                                  self.public_url + "/foo?t=bogus")
1164         return d
1165
1166     def test_GET_DIRURL_json(self):
1167         d = self.GET(self.public_url + "/foo?t=json")
1168         d.addCallback(self.failUnlessIsFooJSON)
1169         return d
1170
1171
1172     def test_POST_DIRURL_manifest_no_ophandle(self):
1173         d = self.shouldFail2(error.Error,
1174                              "test_POST_DIRURL_manifest_no_ophandle",
1175                              "400 Bad Request",
1176                              "slow operation requires ophandle=",
1177                              self.POST, self.public_url, t="start-manifest")
1178         return d
1179
1180     def test_POST_DIRURL_manifest(self):
1181         d = defer.succeed(None)
1182         def getman(ignored, output):
1183             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1184                           followRedirect=True)
1185             d.addCallback(self.wait_for_operation, "125")
1186             d.addCallback(self.get_operation_results, "125", output)
1187             return d
1188         d.addCallback(getman, None)
1189         def _got_html(manifest):
1190             self.failUnless("Manifest of SI=" in manifest)
1191             self.failUnless("<td>sub</td>" in manifest)
1192             self.failUnless(self._sub_uri in manifest)
1193             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1194         d.addCallback(_got_html)
1195
1196         # both t=status and unadorned GET should be identical
1197         d.addCallback(lambda res: self.GET("/operations/125"))
1198         d.addCallback(_got_html)
1199
1200         d.addCallback(getman, "html")
1201         d.addCallback(_got_html)
1202         d.addCallback(getman, "text")
1203         def _got_text(manifest):
1204             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1205             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1206         d.addCallback(_got_text)
1207         d.addCallback(getman, "JSON")
1208         def _got_json(res):
1209             data = res["manifest"]
1210             got = {}
1211             for (path_list, cap) in data:
1212                 got[tuple(path_list)] = cap
1213             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1214             self.failUnless((u"sub",u"baz.txt") in got)
1215             self.failUnless("finished" in res)
1216             self.failUnless("origin" in res)
1217             self.failUnless("storage-index" in res)
1218             self.failUnless("verifycaps" in res)
1219             self.failUnless("stats" in res)
1220         d.addCallback(_got_json)
1221         return d
1222
1223     def test_POST_DIRURL_deepsize_no_ophandle(self):
1224         d = self.shouldFail2(error.Error,
1225                              "test_POST_DIRURL_deepsize_no_ophandle",
1226                              "400 Bad Request",
1227                              "slow operation requires ophandle=",
1228                              self.POST, self.public_url, t="start-deep-size")
1229         return d
1230
1231     def test_POST_DIRURL_deepsize(self):
1232         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1233                       followRedirect=True)
1234         d.addCallback(self.wait_for_operation, "126")
1235         d.addCallback(self.get_operation_results, "126", "json")
1236         def _got_json(data):
1237             self.failUnlessReallyEqual(data["finished"], True)
1238             size = data["size"]
1239             self.failUnless(size > 1000)
1240         d.addCallback(_got_json)
1241         d.addCallback(self.get_operation_results, "126", "text")
1242         def _got_text(res):
1243             mo = re.search(r'^size: (\d+)$', res, re.M)
1244             self.failUnless(mo, res)
1245             size = int(mo.group(1))
1246             # with directories, the size varies.
1247             self.failUnless(size > 1000)
1248         d.addCallback(_got_text)
1249         return d
1250
1251     def test_POST_DIRURL_deepstats_no_ophandle(self):
1252         d = self.shouldFail2(error.Error,
1253                              "test_POST_DIRURL_deepstats_no_ophandle",
1254                              "400 Bad Request",
1255                              "slow operation requires ophandle=",
1256                              self.POST, self.public_url, t="start-deep-stats")
1257         return d
1258
1259     def test_POST_DIRURL_deepstats(self):
1260         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1261                       followRedirect=True)
1262         d.addCallback(self.wait_for_operation, "127")
1263         d.addCallback(self.get_operation_results, "127", "json")
1264         def _got_json(stats):
1265             expected = {"count-immutable-files": 3,
1266                         "count-mutable-files": 0,
1267                         "count-literal-files": 0,
1268                         "count-files": 3,
1269                         "count-directories": 3,
1270                         "size-immutable-files": 57,
1271                         "size-literal-files": 0,
1272                         #"size-directories": 1912, # varies
1273                         #"largest-directory": 1590,
1274                         "largest-directory-children": 5,
1275                         "largest-immutable-file": 19,
1276                         }
1277             for k,v in expected.iteritems():
1278                 self.failUnlessReallyEqual(stats[k], v,
1279                                            "stats[%s] was %s, not %s" %
1280                                            (k, stats[k], v))
1281             self.failUnlessReallyEqual(stats["size-files-histogram"],
1282                                        [ [11, 31, 3] ])
1283         d.addCallback(_got_json)
1284         return d
1285
1286     def test_POST_DIRURL_stream_manifest(self):
1287         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1288         def _check(res):
1289             self.failUnless(res.endswith("\n"))
1290             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1291             self.failUnlessReallyEqual(len(units), 7)
1292             self.failUnlessEqual(units[-1]["type"], "stats")
1293             first = units[0]
1294             self.failUnlessEqual(first["path"], [])
1295             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1296             self.failUnlessEqual(first["type"], "directory")
1297             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1298             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1299             self.failIfEqual(baz["storage-index"], None)
1300             self.failIfEqual(baz["verifycap"], None)
1301             self.failIfEqual(baz["repaircap"], None)
1302             return
1303         d.addCallback(_check)
1304         return d
1305
1306     def test_GET_DIRURL_uri(self):
1307         d = self.GET(self.public_url + "/foo?t=uri")
1308         def _check(res):
1309             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1310         d.addCallback(_check)
1311         return d
1312
1313     def test_GET_DIRURL_readonly_uri(self):
1314         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1315         def _check(res):
1316             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1317         d.addCallback(_check)
1318         return d
1319
1320     def test_PUT_NEWDIRURL(self):
1321         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1322         d.addCallback(lambda res:
1323                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1324         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1325         d.addCallback(self.failUnlessNodeKeysAre, [])
1326         return d
1327
1328     def test_POST_NEWDIRURL(self):
1329         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1330         d.addCallback(lambda res:
1331                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1332         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1333         d.addCallback(self.failUnlessNodeKeysAre, [])
1334         return d
1335
1336     def test_POST_NEWDIRURL_emptyname(self):
1337         # an empty pathname component (i.e. a double-slash) is disallowed
1338         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1339                              "400 Bad Request",
1340                              "The webapi does not allow empty pathname components, i.e. a double slash",
1341                              self.POST, self.public_url + "//?t=mkdir")
1342         return d
1343
1344     def test_POST_NEWDIRURL_initial_children(self):
1345         (newkids, caps) = self._create_initial_children()
1346         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1347                        simplejson.dumps(newkids))
1348         def _check(uri):
1349             n = self.s.create_node_from_uri(uri.strip())
1350             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1351             d2.addCallback(lambda ign:
1352                            self.failUnlessROChildURIIs(n, u"child-imm",
1353                                                        caps['filecap1']))
1354             d2.addCallback(lambda ign:
1355                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1356                                                        caps['filecap2']))
1357             d2.addCallback(lambda ign:
1358                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1359                                                        caps['filecap3']))
1360             d2.addCallback(lambda ign:
1361                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1362                                                        caps['unknown_rocap']))
1363             d2.addCallback(lambda ign:
1364                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1365                                                        caps['unknown_rwcap']))
1366             d2.addCallback(lambda ign:
1367                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1368                                                        caps['unknown_immcap']))
1369             d2.addCallback(lambda ign:
1370                            self.failUnlessRWChildURIIs(n, u"dirchild",
1371                                                        caps['dircap']))
1372             d2.addCallback(lambda ign:
1373                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1374                                                        caps['litdircap']))
1375             d2.addCallback(lambda ign:
1376                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1377                                                        caps['emptydircap']))
1378             return d2
1379         d.addCallback(_check)
1380         d.addCallback(lambda res:
1381                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1382         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1383         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1384         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1385         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1386         return d
1387
1388     def test_POST_NEWDIRURL_immutable(self):
1389         (newkids, caps) = self._create_immutable_children()
1390         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1391                        simplejson.dumps(newkids))
1392         def _check(uri):
1393             n = self.s.create_node_from_uri(uri.strip())
1394             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1395             d2.addCallback(lambda ign:
1396                            self.failUnlessROChildURIIs(n, u"child-imm",
1397                                                        caps['filecap1']))
1398             d2.addCallback(lambda ign:
1399                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1400                                                        caps['unknown_immcap']))
1401             d2.addCallback(lambda ign:
1402                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1403                                                        caps['immdircap']))
1404             d2.addCallback(lambda ign:
1405                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1406                                                        caps['litdircap']))
1407             d2.addCallback(lambda ign:
1408                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1409                                                        caps['emptydircap']))
1410             return d2
1411         d.addCallback(_check)
1412         d.addCallback(lambda res:
1413                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1414         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1415         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1416         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1417         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1418         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1419         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1420         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1421         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1422         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1423         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1424         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1425         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1426         d.addErrback(self.explain_web_error)
1427         return d
1428
1429     def test_POST_NEWDIRURL_immutable_bad(self):
1430         (newkids, caps) = self._create_initial_children()
1431         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1432                              "400 Bad Request",
1433                              "needed to be immutable but was not",
1434                              self.POST2,
1435                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1436                              simplejson.dumps(newkids))
1437         return d
1438
1439     def test_PUT_NEWDIRURL_exists(self):
1440         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1441         d.addCallback(lambda res:
1442                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1443         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1444         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1445         return d
1446
1447     def test_PUT_NEWDIRURL_blocked(self):
1448         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1449                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1450                              self.PUT,
1451                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1452         d.addCallback(lambda res:
1453                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1454         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1455         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1456         return d
1457
1458     def test_PUT_NEWDIRURL_mkdir_p(self):
1459         d = defer.succeed(None)
1460         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1461         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1462         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1463         def mkdir_p(mkpnode):
1464             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1465             d = self.POST(url)
1466             def made_subsub(ssuri):
1467                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1468                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1469                 d = self.POST(url)
1470                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1471                 return d
1472             d.addCallback(made_subsub)
1473             return d
1474         d.addCallback(mkdir_p)
1475         return d
1476
1477     def test_PUT_NEWDIRURL_mkdirs(self):
1478         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1479         d.addCallback(lambda res:
1480                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1481         d.addCallback(lambda res:
1482                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1483         d.addCallback(lambda res:
1484                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1485         d.addCallback(self.failUnlessNodeKeysAre, [])
1486         return d
1487
1488     def test_DELETE_DIRURL(self):
1489         d = self.DELETE(self.public_url + "/foo")
1490         d.addCallback(lambda res:
1491                       self.failIfNodeHasChild(self.public_root, u"foo"))
1492         return d
1493
1494     def test_DELETE_DIRURL_missing(self):
1495         d = self.DELETE(self.public_url + "/foo/missing")
1496         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1497         d.addCallback(lambda res:
1498                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1499         return d
1500
1501     def test_DELETE_DIRURL_missing2(self):
1502         d = self.DELETE(self.public_url + "/missing")
1503         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1504         return d
1505
1506     def dump_root(self):
1507         print "NODEWALK"
1508         w = webish.DirnodeWalkerMixin()
1509         def visitor(childpath, childnode, metadata):
1510             print childpath
1511         d = w.walk(self.public_root, visitor)
1512         return d
1513
1514     def failUnlessNodeKeysAre(self, node, expected_keys):
1515         for k in expected_keys:
1516             assert isinstance(k, unicode)
1517         d = node.list()
1518         def _check(children):
1519             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1520         d.addCallback(_check)
1521         return d
1522     def failUnlessNodeHasChild(self, node, name):
1523         assert isinstance(name, unicode)
1524         d = node.list()
1525         def _check(children):
1526             self.failUnless(name in children)
1527         d.addCallback(_check)
1528         return d
1529     def failIfNodeHasChild(self, node, name):
1530         assert isinstance(name, unicode)
1531         d = node.list()
1532         def _check(children):
1533             self.failIf(name in children)
1534         d.addCallback(_check)
1535         return d
1536
1537     def failUnlessChildContentsAre(self, node, name, expected_contents):
1538         assert isinstance(name, unicode)
1539         d = node.get_child_at_path(name)
1540         d.addCallback(lambda node: download_to_data(node))
1541         def _check(contents):
1542             self.failUnlessReallyEqual(contents, expected_contents)
1543         d.addCallback(_check)
1544         return d
1545
1546     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1547         assert isinstance(name, unicode)
1548         d = node.get_child_at_path(name)
1549         d.addCallback(lambda node: node.download_best_version())
1550         def _check(contents):
1551             self.failUnlessReallyEqual(contents, expected_contents)
1552         d.addCallback(_check)
1553         return d
1554
1555     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1556         assert isinstance(name, unicode)
1557         d = node.get_child_at_path(name)
1558         def _check(child):
1559             self.failUnless(child.is_unknown() or not child.is_readonly())
1560             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1561             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1562             expected_ro_uri = self._make_readonly(expected_uri)
1563             if expected_ro_uri:
1564                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1565         d.addCallback(_check)
1566         return d
1567
1568     def failUnlessROChildURIIs(self, node, name, expected_uri):
1569         assert isinstance(name, unicode)
1570         d = node.get_child_at_path(name)
1571         def _check(child):
1572             self.failUnless(child.is_unknown() or child.is_readonly())
1573             self.failUnlessReallyEqual(child.get_write_uri(), None)
1574             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1575             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1576         d.addCallback(_check)
1577         return d
1578
1579     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1580         assert isinstance(name, unicode)
1581         d = node.get_child_at_path(name)
1582         def _check(child):
1583             self.failUnless(child.is_unknown() or not child.is_readonly())
1584             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1585             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1586             expected_ro_uri = self._make_readonly(got_uri)
1587             if expected_ro_uri:
1588                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1589         d.addCallback(_check)
1590         return d
1591
1592     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1593         assert isinstance(name, unicode)
1594         d = node.get_child_at_path(name)
1595         def _check(child):
1596             self.failUnless(child.is_unknown() or child.is_readonly())
1597             self.failUnlessReallyEqual(child.get_write_uri(), None)
1598             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1599             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1600         d.addCallback(_check)
1601         return d
1602
1603     def failUnlessCHKURIHasContents(self, got_uri, contents):
1604         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1605
1606     def test_POST_upload(self):
1607         d = self.POST(self.public_url + "/foo", t="upload",
1608                       file=("new.txt", self.NEWFILE_CONTENTS))
1609         fn = self._foo_node
1610         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1611         d.addCallback(lambda res:
1612                       self.failUnlessChildContentsAre(fn, u"new.txt",
1613                                                       self.NEWFILE_CONTENTS))
1614         return d
1615
1616     def test_POST_upload_unicode(self):
1617         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1618         d = self.POST(self.public_url + "/foo", t="upload",
1619                       file=(filename, self.NEWFILE_CONTENTS))
1620         fn = self._foo_node
1621         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1622         d.addCallback(lambda res:
1623                       self.failUnlessChildContentsAre(fn, filename,
1624                                                       self.NEWFILE_CONTENTS))
1625         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1626         d.addCallback(lambda res: self.GET(target_url))
1627         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1628                                                                   self.NEWFILE_CONTENTS,
1629                                                                   contents))
1630         return d
1631
1632     def test_POST_upload_unicode_named(self):
1633         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1634         d = self.POST(self.public_url + "/foo", t="upload",
1635                       name=filename,
1636                       file=("overridden", self.NEWFILE_CONTENTS))
1637         fn = self._foo_node
1638         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1639         d.addCallback(lambda res:
1640                       self.failUnlessChildContentsAre(fn, filename,
1641                                                       self.NEWFILE_CONTENTS))
1642         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1643         d.addCallback(lambda res: self.GET(target_url))
1644         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1645                                                                   self.NEWFILE_CONTENTS,
1646                                                                   contents))
1647         return d
1648
1649     def test_POST_upload_no_link(self):
1650         d = self.POST("/uri", t="upload",
1651                       file=("new.txt", self.NEWFILE_CONTENTS))
1652         def _check_upload_results(page):
1653             # this should be a page which describes the results of the upload
1654             # that just finished.
1655             self.failUnless("Upload Results:" in page)
1656             self.failUnless("URI:" in page)
1657             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1658             mo = uri_re.search(page)
1659             self.failUnless(mo, page)
1660             new_uri = mo.group(1)
1661             return new_uri
1662         d.addCallback(_check_upload_results)
1663         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1664         return d
1665
1666     def test_POST_upload_no_link_whendone(self):
1667         d = self.POST("/uri", t="upload", when_done="/",
1668                       file=("new.txt", self.NEWFILE_CONTENTS))
1669         d.addBoth(self.shouldRedirect, "/")
1670         return d
1671
1672     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1673         d = defer.maybeDeferred(callable, *args, **kwargs)
1674         def done(res):
1675             if isinstance(res, failure.Failure):
1676                 res.trap(error.PageRedirect)
1677                 statuscode = res.value.status
1678                 target = res.value.location
1679                 return checker(statuscode, target)
1680             self.fail("%s: callable was supposed to redirect, not return '%s'"
1681                       % (which, res))
1682         d.addBoth(done)
1683         return d
1684
1685     def test_POST_upload_no_link_whendone_results(self):
1686         def check(statuscode, target):
1687             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1688             self.failUnless(target.startswith(self.webish_url), target)
1689             return client.getPage(target, method="GET")
1690         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1691                                  check,
1692                                  self.POST, "/uri", t="upload",
1693                                  when_done="/uri/%(uri)s",
1694                                  file=("new.txt", self.NEWFILE_CONTENTS))
1695         d.addCallback(lambda res:
1696                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1697         return d
1698
1699     def test_POST_upload_no_link_mutable(self):
1700         d = self.POST("/uri", t="upload", mutable="true",
1701                       file=("new.txt", self.NEWFILE_CONTENTS))
1702         def _check(filecap):
1703             filecap = filecap.strip()
1704             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1705             self.filecap = filecap
1706             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1707             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1708             n = self.s.create_node_from_uri(filecap)
1709             return n.download_best_version()
1710         d.addCallback(_check)
1711         def _check2(data):
1712             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1713             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1714         d.addCallback(_check2)
1715         def _check3(data):
1716             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1717             return self.GET("/file/%s" % urllib.quote(self.filecap))
1718         d.addCallback(_check3)
1719         def _check4(data):
1720             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1721         d.addCallback(_check4)
1722         return d
1723
1724     def test_POST_upload_no_link_mutable_toobig(self):
1725         d = self.shouldFail2(error.Error,
1726                              "test_POST_upload_no_link_mutable_toobig",
1727                              "413 Request Entity Too Large",
1728                              "SDMF is limited to one segment, and 10001 > 10000",
1729                              self.POST,
1730                              "/uri", t="upload", mutable="true",
1731                              file=("new.txt",
1732                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1733         return d
1734
1735     def test_POST_upload_mutable(self):
1736         # this creates a mutable file
1737         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1738                       file=("new.txt", self.NEWFILE_CONTENTS))
1739         fn = self._foo_node
1740         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1741         d.addCallback(lambda res:
1742                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1743                                                              self.NEWFILE_CONTENTS))
1744         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1745         def _got(newnode):
1746             self.failUnless(IMutableFileNode.providedBy(newnode))
1747             self.failUnless(newnode.is_mutable())
1748             self.failIf(newnode.is_readonly())
1749             self._mutable_node = newnode
1750             self._mutable_uri = newnode.get_uri()
1751         d.addCallback(_got)
1752
1753         # now upload it again and make sure that the URI doesn't change
1754         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1755         d.addCallback(lambda res:
1756                       self.POST(self.public_url + "/foo", t="upload",
1757                                 mutable="true",
1758                                 file=("new.txt", NEWER_CONTENTS)))
1759         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1760         d.addCallback(lambda res:
1761                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1762                                                              NEWER_CONTENTS))
1763         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1764         def _got2(newnode):
1765             self.failUnless(IMutableFileNode.providedBy(newnode))
1766             self.failUnless(newnode.is_mutable())
1767             self.failIf(newnode.is_readonly())
1768             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1769         d.addCallback(_got2)
1770
1771         # upload a second time, using PUT instead of POST
1772         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1773         d.addCallback(lambda res:
1774                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1775         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1776         d.addCallback(lambda res:
1777                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1778                                                              NEW2_CONTENTS))
1779
1780         # finally list the directory, since mutable files are displayed
1781         # slightly differently
1782
1783         d.addCallback(lambda res:
1784                       self.GET(self.public_url + "/foo/",
1785                                followRedirect=True))
1786         def _check_page(res):
1787             # TODO: assert more about the contents
1788             self.failUnless("SSK" in res)
1789             return res
1790         d.addCallback(_check_page)
1791
1792         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1793         def _got3(newnode):
1794             self.failUnless(IMutableFileNode.providedBy(newnode))
1795             self.failUnless(newnode.is_mutable())
1796             self.failIf(newnode.is_readonly())
1797             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1798         d.addCallback(_got3)
1799
1800         # look at the JSON form of the enclosing directory
1801         d.addCallback(lambda res:
1802                       self.GET(self.public_url + "/foo/?t=json",
1803                                followRedirect=True))
1804         def _check_page_json(res):
1805             parsed = simplejson.loads(res)
1806             self.failUnlessEqual(parsed[0], "dirnode")
1807             children = dict( [(unicode(name),value)
1808                               for (name,value)
1809                               in parsed[1]["children"].iteritems()] )
1810             self.failUnless(u"new.txt" in children)
1811             new_json = children[u"new.txt"]
1812             self.failUnlessEqual(new_json[0], "filenode")
1813             self.failUnless(new_json[1]["mutable"])
1814             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
1815             ro_uri = self._mutable_node.get_readonly().to_string()
1816             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
1817         d.addCallback(_check_page_json)
1818
1819         # and the JSON form of the file
1820         d.addCallback(lambda res:
1821                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1822         def _check_file_json(res):
1823             parsed = simplejson.loads(res)
1824             self.failUnlessEqual(parsed[0], "filenode")
1825             self.failUnless(parsed[1]["mutable"])
1826             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
1827             ro_uri = self._mutable_node.get_readonly().to_string()
1828             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
1829         d.addCallback(_check_file_json)
1830
1831         # and look at t=uri and t=readonly-uri
1832         d.addCallback(lambda res:
1833                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1834         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1835         d.addCallback(lambda res:
1836                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1837         def _check_ro_uri(res):
1838             ro_uri = self._mutable_node.get_readonly().to_string()
1839             self.failUnlessReallyEqual(res, ro_uri)
1840         d.addCallback(_check_ro_uri)
1841
1842         # make sure we can get to it from /uri/URI
1843         d.addCallback(lambda res:
1844                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1845         d.addCallback(lambda res:
1846                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1847
1848         # and that HEAD computes the size correctly
1849         d.addCallback(lambda res:
1850                       self.HEAD(self.public_url + "/foo/new.txt",
1851                                 return_response=True))
1852         def _got_headers((res, status, headers)):
1853             self.failUnlessReallyEqual(res, "")
1854             self.failUnlessReallyEqual(headers["content-length"][0],
1855                                        str(len(NEW2_CONTENTS)))
1856             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1857         d.addCallback(_got_headers)
1858
1859         # make sure that size errors are displayed correctly for overwrite
1860         d.addCallback(lambda res:
1861                       self.shouldFail2(error.Error,
1862                                        "test_POST_upload_mutable-toobig",
1863                                        "413 Request Entity Too Large",
1864                                        "SDMF is limited to one segment, and 10001 > 10000",
1865                                        self.POST,
1866                                        self.public_url + "/foo", t="upload",
1867                                        mutable="true",
1868                                        file=("new.txt",
1869                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1870                                        ))
1871
1872         d.addErrback(self.dump_error)
1873         return d
1874
1875     def test_POST_upload_mutable_toobig(self):
1876         d = self.shouldFail2(error.Error,
1877                              "test_POST_upload_mutable_toobig",
1878                              "413 Request Entity Too Large",
1879                              "SDMF is limited to one segment, and 10001 > 10000",
1880                              self.POST,
1881                              self.public_url + "/foo",
1882                              t="upload", mutable="true",
1883                              file=("new.txt",
1884                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1885         return d
1886
1887     def dump_error(self, f):
1888         # if the web server returns an error code (like 400 Bad Request),
1889         # web.client.getPage puts the HTTP response body into the .response
1890         # attribute of the exception object that it gives back. It does not
1891         # appear in the Failure's repr(), so the ERROR that trial displays
1892         # will be rather terse and unhelpful. addErrback this method to the
1893         # end of your chain to get more information out of these errors.
1894         if f.check(error.Error):
1895             print "web.error.Error:"
1896             print f
1897             print f.value.response
1898         return f
1899
1900     def test_POST_upload_replace(self):
1901         d = self.POST(self.public_url + "/foo", t="upload",
1902                       file=("bar.txt", self.NEWFILE_CONTENTS))
1903         fn = self._foo_node
1904         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1905         d.addCallback(lambda res:
1906                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1907                                                       self.NEWFILE_CONTENTS))
1908         return d
1909
1910     def test_POST_upload_no_replace_ok(self):
1911         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1912                       file=("new.txt", self.NEWFILE_CONTENTS))
1913         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1914         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1915                                                              self.NEWFILE_CONTENTS))
1916         return d
1917
1918     def test_POST_upload_no_replace_queryarg(self):
1919         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1920                       file=("bar.txt", self.NEWFILE_CONTENTS))
1921         d.addBoth(self.shouldFail, error.Error,
1922                   "POST_upload_no_replace_queryarg",
1923                   "409 Conflict",
1924                   "There was already a child by that name, and you asked me "
1925                   "to not replace it")
1926         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1927         d.addCallback(self.failUnlessIsBarDotTxt)
1928         return d
1929
1930     def test_POST_upload_no_replace_field(self):
1931         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1932                       file=("bar.txt", self.NEWFILE_CONTENTS))
1933         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1934                   "409 Conflict",
1935                   "There was already a child by that name, and you asked me "
1936                   "to not replace it")
1937         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1938         d.addCallback(self.failUnlessIsBarDotTxt)
1939         return d
1940
1941     def test_POST_upload_whendone(self):
1942         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1943                       file=("new.txt", self.NEWFILE_CONTENTS))
1944         d.addBoth(self.shouldRedirect, "/THERE")
1945         fn = self._foo_node
1946         d.addCallback(lambda res:
1947                       self.failUnlessChildContentsAre(fn, u"new.txt",
1948                                                       self.NEWFILE_CONTENTS))
1949         return d
1950
1951     def test_POST_upload_named(self):
1952         fn = self._foo_node
1953         d = self.POST(self.public_url + "/foo", t="upload",
1954                       name="new.txt", file=self.NEWFILE_CONTENTS)
1955         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1956         d.addCallback(lambda res:
1957                       self.failUnlessChildContentsAre(fn, u"new.txt",
1958                                                       self.NEWFILE_CONTENTS))
1959         return d
1960
1961     def test_POST_upload_named_badfilename(self):
1962         d = self.POST(self.public_url + "/foo", t="upload",
1963                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1964         d.addBoth(self.shouldFail, error.Error,
1965                   "test_POST_upload_named_badfilename",
1966                   "400 Bad Request",
1967                   "name= may not contain a slash",
1968                   )
1969         # make sure that nothing was added
1970         d.addCallback(lambda res:
1971                       self.failUnlessNodeKeysAre(self._foo_node,
1972                                                  [u"bar.txt", u"blockingfile",
1973                                                   u"empty", u"n\u00fc.txt",
1974                                                   u"sub"]))
1975         return d
1976
1977     def test_POST_FILEURL_check(self):
1978         bar_url = self.public_url + "/foo/bar.txt"
1979         d = self.POST(bar_url, t="check")
1980         def _check(res):
1981             self.failUnless("Healthy :" in res)
1982         d.addCallback(_check)
1983         redir_url = "http://allmydata.org/TARGET"
1984         def _check2(statuscode, target):
1985             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1986             self.failUnlessReallyEqual(target, redir_url)
1987         d.addCallback(lambda res:
1988                       self.shouldRedirect2("test_POST_FILEURL_check",
1989                                            _check2,
1990                                            self.POST, bar_url,
1991                                            t="check",
1992                                            when_done=redir_url))
1993         d.addCallback(lambda res:
1994                       self.POST(bar_url, t="check", return_to=redir_url))
1995         def _check3(res):
1996             self.failUnless("Healthy :" in res)
1997             self.failUnless("Return to file" in res)
1998             self.failUnless(redir_url in res)
1999         d.addCallback(_check3)
2000
2001         d.addCallback(lambda res:
2002                       self.POST(bar_url, t="check", output="JSON"))
2003         def _check_json(res):
2004             data = simplejson.loads(res)
2005             self.failUnless("storage-index" in data)
2006             self.failUnless(data["results"]["healthy"])
2007         d.addCallback(_check_json)
2008
2009         return d
2010
2011     def test_POST_FILEURL_check_and_repair(self):
2012         bar_url = self.public_url + "/foo/bar.txt"
2013         d = self.POST(bar_url, t="check", repair="true")
2014         def _check(res):
2015             self.failUnless("Healthy :" in res)
2016         d.addCallback(_check)
2017         redir_url = "http://allmydata.org/TARGET"
2018         def _check2(statuscode, target):
2019             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2020             self.failUnlessReallyEqual(target, redir_url)
2021         d.addCallback(lambda res:
2022                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2023                                            _check2,
2024                                            self.POST, bar_url,
2025                                            t="check", repair="true",
2026                                            when_done=redir_url))
2027         d.addCallback(lambda res:
2028                       self.POST(bar_url, t="check", return_to=redir_url))
2029         def _check3(res):
2030             self.failUnless("Healthy :" in res)
2031             self.failUnless("Return to file" in res)
2032             self.failUnless(redir_url in res)
2033         d.addCallback(_check3)
2034         return d
2035
2036     def test_POST_DIRURL_check(self):
2037         foo_url = self.public_url + "/foo/"
2038         d = self.POST(foo_url, t="check")
2039         def _check(res):
2040             self.failUnless("Healthy :" in res, res)
2041         d.addCallback(_check)
2042         redir_url = "http://allmydata.org/TARGET"
2043         def _check2(statuscode, target):
2044             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2045             self.failUnlessReallyEqual(target, redir_url)
2046         d.addCallback(lambda res:
2047                       self.shouldRedirect2("test_POST_DIRURL_check",
2048                                            _check2,
2049                                            self.POST, foo_url,
2050                                            t="check",
2051                                            when_done=redir_url))
2052         d.addCallback(lambda res:
2053                       self.POST(foo_url, t="check", return_to=redir_url))
2054         def _check3(res):
2055             self.failUnless("Healthy :" in res, res)
2056             self.failUnless("Return to file/directory" in res)
2057             self.failUnless(redir_url in res)
2058         d.addCallback(_check3)
2059
2060         d.addCallback(lambda res:
2061                       self.POST(foo_url, t="check", output="JSON"))
2062         def _check_json(res):
2063             data = simplejson.loads(res)
2064             self.failUnless("storage-index" in data)
2065             self.failUnless(data["results"]["healthy"])
2066         d.addCallback(_check_json)
2067
2068         return d
2069
2070     def test_POST_DIRURL_check_and_repair(self):
2071         foo_url = self.public_url + "/foo/"
2072         d = self.POST(foo_url, t="check", repair="true")
2073         def _check(res):
2074             self.failUnless("Healthy :" in res, res)
2075         d.addCallback(_check)
2076         redir_url = "http://allmydata.org/TARGET"
2077         def _check2(statuscode, target):
2078             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2079             self.failUnlessReallyEqual(target, redir_url)
2080         d.addCallback(lambda res:
2081                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2082                                            _check2,
2083                                            self.POST, foo_url,
2084                                            t="check", repair="true",
2085                                            when_done=redir_url))
2086         d.addCallback(lambda res:
2087                       self.POST(foo_url, t="check", return_to=redir_url))
2088         def _check3(res):
2089             self.failUnless("Healthy :" in res)
2090             self.failUnless("Return to file/directory" in res)
2091             self.failUnless(redir_url in res)
2092         d.addCallback(_check3)
2093         return d
2094
2095     def wait_for_operation(self, ignored, ophandle):
2096         url = "/operations/" + ophandle
2097         url += "?t=status&output=JSON"
2098         d = self.GET(url)
2099         def _got(res):
2100             data = simplejson.loads(res)
2101             if not data["finished"]:
2102                 d = self.stall(delay=1.0)
2103                 d.addCallback(self.wait_for_operation, ophandle)
2104                 return d
2105             return data
2106         d.addCallback(_got)
2107         return d
2108
2109     def get_operation_results(self, ignored, ophandle, output=None):
2110         url = "/operations/" + ophandle
2111         url += "?t=status"
2112         if output:
2113             url += "&output=" + output
2114         d = self.GET(url)
2115         def _got(res):
2116             if output and output.lower() == "json":
2117                 return simplejson.loads(res)
2118             return res
2119         d.addCallback(_got)
2120         return d
2121
2122     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2123         d = self.shouldFail2(error.Error,
2124                              "test_POST_DIRURL_deepcheck_no_ophandle",
2125                              "400 Bad Request",
2126                              "slow operation requires ophandle=",
2127                              self.POST, self.public_url, t="start-deep-check")
2128         return d
2129
2130     def test_POST_DIRURL_deepcheck(self):
2131         def _check_redirect(statuscode, target):
2132             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2133             self.failUnless(target.endswith("/operations/123"))
2134         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2135                                  self.POST, self.public_url,
2136                                  t="start-deep-check", ophandle="123")
2137         d.addCallback(self.wait_for_operation, "123")
2138         def _check_json(data):
2139             self.failUnlessReallyEqual(data["finished"], True)
2140             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2141             self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2142         d.addCallback(_check_json)
2143         d.addCallback(self.get_operation_results, "123", "html")
2144         def _check_html(res):
2145             self.failUnless("Objects Checked: <span>8</span>" in res)
2146             self.failUnless("Objects Healthy: <span>8</span>" in res)
2147         d.addCallback(_check_html)
2148
2149         d.addCallback(lambda res:
2150                       self.GET("/operations/123/"))
2151         d.addCallback(_check_html) # should be the same as without the slash
2152
2153         d.addCallback(lambda res:
2154                       self.shouldFail2(error.Error, "one", "404 Not Found",
2155                                        "No detailed results for SI bogus",
2156                                        self.GET, "/operations/123/bogus"))
2157
2158         foo_si = self._foo_node.get_storage_index()
2159         foo_si_s = base32.b2a(foo_si)
2160         d.addCallback(lambda res:
2161                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2162         def _check_foo_json(res):
2163             data = simplejson.loads(res)
2164             self.failUnlessEqual(data["storage-index"], foo_si_s)
2165             self.failUnless(data["results"]["healthy"])
2166         d.addCallback(_check_foo_json)
2167         return d
2168
2169     def test_POST_DIRURL_deepcheck_and_repair(self):
2170         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2171                       ophandle="124", output="json", followRedirect=True)
2172         d.addCallback(self.wait_for_operation, "124")
2173         def _check_json(data):
2174             self.failUnlessReallyEqual(data["finished"], True)
2175             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2176             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2177             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2178             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2179             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2180             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2181             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2182             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2183             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2184             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2185         d.addCallback(_check_json)
2186         d.addCallback(self.get_operation_results, "124", "html")
2187         def _check_html(res):
2188             self.failUnless("Objects Checked: <span>8</span>" in res)
2189
2190             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2191             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2192             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2193
2194             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2195             self.failUnless("Repairs Successful: <span>0</span>" in res)
2196             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2197
2198             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2199             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2200             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2201         d.addCallback(_check_html)
2202         return d
2203
2204     def test_POST_FILEURL_bad_t(self):
2205         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2206                              "POST to file: bad t=bogus",
2207                              self.POST, self.public_url + "/foo/bar.txt",
2208                              t="bogus")
2209         return d
2210
2211     def test_POST_mkdir(self): # return value?
2212         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2213         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2214         d.addCallback(self.failUnlessNodeKeysAre, [])
2215         return d
2216
2217     def test_POST_mkdir_initial_children(self):
2218         (newkids, caps) = self._create_initial_children()
2219         d = self.POST2(self.public_url +
2220                        "/foo?t=mkdir-with-children&name=newdir",
2221                        simplejson.dumps(newkids))
2222         d.addCallback(lambda res:
2223                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2224         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2225         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2226         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2227         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2228         return d
2229
2230     def test_POST_mkdir_immutable(self):
2231         (newkids, caps) = self._create_immutable_children()
2232         d = self.POST2(self.public_url +
2233                        "/foo?t=mkdir-immutable&name=newdir",
2234                        simplejson.dumps(newkids))
2235         d.addCallback(lambda res:
2236                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2237         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2238         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2239         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2240         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2241         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2242         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2243         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2244         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2245         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2246         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2247         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2248         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2249         return d
2250
2251     def test_POST_mkdir_immutable_bad(self):
2252         (newkids, caps) = self._create_initial_children()
2253         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2254                              "400 Bad Request",
2255                              "needed to be immutable but was not",
2256                              self.POST2,
2257                              self.public_url +
2258                              "/foo?t=mkdir-immutable&name=newdir",
2259                              simplejson.dumps(newkids))
2260         return d
2261
2262     def test_POST_mkdir_2(self):
2263         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2264         d.addCallback(lambda res:
2265                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2266         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2267         d.addCallback(self.failUnlessNodeKeysAre, [])
2268         return d
2269
2270     def test_POST_mkdirs_2(self):
2271         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2272         d.addCallback(lambda res:
2273                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2274         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2275         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2276         d.addCallback(self.failUnlessNodeKeysAre, [])
2277         return d
2278
2279     def test_POST_mkdir_no_parentdir_noredirect(self):
2280         d = self.POST("/uri?t=mkdir")
2281         def _after_mkdir(res):
2282             uri.DirectoryURI.init_from_string(res)
2283         d.addCallback(_after_mkdir)
2284         return d
2285
2286     def test_POST_mkdir_no_parentdir_noredirect2(self):
2287         # make sure form-based arguments (as on the welcome page) still work
2288         d = self.POST("/uri", t="mkdir")
2289         def _after_mkdir(res):
2290             uri.DirectoryURI.init_from_string(res)
2291         d.addCallback(_after_mkdir)
2292         d.addErrback(self.explain_web_error)
2293         return d
2294
2295     def test_POST_mkdir_no_parentdir_redirect(self):
2296         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2297         d.addBoth(self.shouldRedirect, None, statuscode='303')
2298         def _check_target(target):
2299             target = urllib.unquote(target)
2300             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2301         d.addCallback(_check_target)
2302         return d
2303
2304     def test_POST_mkdir_no_parentdir_redirect2(self):
2305         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2306         d.addBoth(self.shouldRedirect, None, statuscode='303')
2307         def _check_target(target):
2308             target = urllib.unquote(target)
2309             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2310         d.addCallback(_check_target)
2311         d.addErrback(self.explain_web_error)
2312         return d
2313
2314     def _make_readonly(self, u):
2315         ro_uri = uri.from_string(u).get_readonly()
2316         if ro_uri is None:
2317             return None
2318         return ro_uri.to_string()
2319
2320     def _create_initial_children(self):
2321         contents, n, filecap1 = self.makefile(12)
2322         md1 = {"metakey1": "metavalue1"}
2323         filecap2 = make_mutable_file_uri()
2324         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2325         filecap3 = node3.get_readonly_uri()
2326         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2327         dircap = DirectoryNode(node4, None, None).get_uri()
2328         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2329         emptydircap = "URI:DIR2-LIT:"
2330         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2331                                                       "ro_uri": self._make_readonly(filecap1),
2332                                                       "metadata": md1, }],
2333                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2334                                                       "ro_uri": self._make_readonly(filecap2)}],
2335                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2336                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2337                                                       "ro_uri": unknown_rocap}],
2338                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2339                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2340                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2341                                                       "ro_uri": self._make_readonly(dircap)}],
2342                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2343                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2344                    }
2345         return newkids, {'filecap1': filecap1,
2346                          'filecap2': filecap2,
2347                          'filecap3': filecap3,
2348                          'unknown_rwcap': unknown_rwcap,
2349                          'unknown_rocap': unknown_rocap,
2350                          'unknown_immcap': unknown_immcap,
2351                          'dircap': dircap,
2352                          'litdircap': litdircap,
2353                          'emptydircap': emptydircap}
2354
2355     def _create_immutable_children(self):
2356         contents, n, filecap1 = self.makefile(12)
2357         md1 = {"metakey1": "metavalue1"}
2358         tnode = create_chk_filenode("immutable directory contents\n"*10)
2359         dnode = DirectoryNode(tnode, None, None)
2360         assert not dnode.is_mutable()
2361         immdircap = dnode.get_uri()
2362         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2363         emptydircap = "URI:DIR2-LIT:"
2364         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2365                                                       "metadata": md1, }],
2366                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2367                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2368                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2369                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2370                    }
2371         return newkids, {'filecap1': filecap1,
2372                          'unknown_immcap': unknown_immcap,
2373                          'immdircap': immdircap,
2374                          'litdircap': litdircap,
2375                          'emptydircap': emptydircap}
2376
2377     def test_POST_mkdir_no_parentdir_initial_children(self):
2378         (newkids, caps) = self._create_initial_children()
2379         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2380         def _after_mkdir(res):
2381             self.failUnless(res.startswith("URI:DIR"), res)
2382             n = self.s.create_node_from_uri(res)
2383             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2384             d2.addCallback(lambda ign:
2385                            self.failUnlessROChildURIIs(n, u"child-imm",
2386                                                        caps['filecap1']))
2387             d2.addCallback(lambda ign:
2388                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2389                                                        caps['filecap2']))
2390             d2.addCallback(lambda ign:
2391                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2392                                                        caps['filecap3']))
2393             d2.addCallback(lambda ign:
2394                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2395                                                        caps['unknown_rwcap']))
2396             d2.addCallback(lambda ign:
2397                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2398                                                        caps['unknown_rocap']))
2399             d2.addCallback(lambda ign:
2400                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2401                                                        caps['unknown_immcap']))
2402             d2.addCallback(lambda ign:
2403                            self.failUnlessRWChildURIIs(n, u"dirchild",
2404                                                        caps['dircap']))
2405             return d2
2406         d.addCallback(_after_mkdir)
2407         return d
2408
2409     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2410         # the regular /uri?t=mkdir operation is specified to ignore its body.
2411         # Only t=mkdir-with-children pays attention to it.
2412         (newkids, caps) = self._create_initial_children()
2413         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2414                                  400, "Bad Request",
2415                                  "t=mkdir does not accept children=, "
2416                                  "try t=mkdir-with-children instead",
2417                                  self.POST2, "/uri?t=mkdir", # without children
2418                                  simplejson.dumps(newkids))
2419         return d
2420
2421     def test_POST_noparent_bad(self):
2422         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2423                                  "/uri accepts only PUT, PUT?t=mkdir, "
2424                                  "POST?t=upload, and POST?t=mkdir",
2425                                  self.POST, "/uri?t=bogus")
2426         return d
2427
2428     def test_POST_mkdir_no_parentdir_immutable(self):
2429         (newkids, caps) = self._create_immutable_children()
2430         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2431         def _after_mkdir(res):
2432             self.failUnless(res.startswith("URI:DIR"), res)
2433             n = self.s.create_node_from_uri(res)
2434             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2435             d2.addCallback(lambda ign:
2436                            self.failUnlessROChildURIIs(n, u"child-imm",
2437                                                           caps['filecap1']))
2438             d2.addCallback(lambda ign:
2439                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2440                                                           caps['unknown_immcap']))
2441             d2.addCallback(lambda ign:
2442                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2443                                                           caps['immdircap']))
2444             d2.addCallback(lambda ign:
2445                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2446                                                           caps['litdircap']))
2447             d2.addCallback(lambda ign:
2448                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2449                                                           caps['emptydircap']))
2450             return d2
2451         d.addCallback(_after_mkdir)
2452         return d
2453
2454     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2455         (newkids, caps) = self._create_initial_children()
2456         d = self.shouldFail2(error.Error,
2457                              "test_POST_mkdir_no_parentdir_immutable_bad",
2458                              "400 Bad Request",
2459                              "needed to be immutable but was not",
2460                              self.POST2,
2461                              "/uri?t=mkdir-immutable",
2462                              simplejson.dumps(newkids))
2463         return d
2464
2465     def test_welcome_page_mkdir_button(self):
2466         # Fetch the welcome page.
2467         d = self.GET("/")
2468         def _after_get_welcome_page(res):
2469             MKDIR_BUTTON_RE = re.compile(
2470                 '<form action="([^"]*)" method="post".*?'
2471                 '<input type="hidden" name="t" value="([^"]*)" />'
2472                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2473                 '<input type="submit" value="Create a directory" />',
2474                 re.I)
2475             mo = MKDIR_BUTTON_RE.search(res)
2476             formaction = mo.group(1)
2477             formt = mo.group(2)
2478             formaname = mo.group(3)
2479             formavalue = mo.group(4)
2480             return (formaction, formt, formaname, formavalue)
2481         d.addCallback(_after_get_welcome_page)
2482         def _after_parse_form(res):
2483             (formaction, formt, formaname, formavalue) = res
2484             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2485         d.addCallback(_after_parse_form)
2486         d.addBoth(self.shouldRedirect, None, statuscode='303')
2487         return d
2488
2489     def test_POST_mkdir_replace(self): # return value?
2490         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2491         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2492         d.addCallback(self.failUnlessNodeKeysAre, [])
2493         return d
2494
2495     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2496         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2497         d.addBoth(self.shouldFail, error.Error,
2498                   "POST_mkdir_no_replace_queryarg",
2499                   "409 Conflict",
2500                   "There was already a child by that name, and you asked me "
2501                   "to not replace it")
2502         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2503         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2504         return d
2505
2506     def test_POST_mkdir_no_replace_field(self): # return value?
2507         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2508                       replace="false")
2509         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2510                   "409 Conflict",
2511                   "There was already a child by that name, and you asked me "
2512                   "to not replace it")
2513         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2514         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2515         return d
2516
2517     def test_POST_mkdir_whendone_field(self):
2518         d = self.POST(self.public_url + "/foo",
2519                       t="mkdir", name="newdir", when_done="/THERE")
2520         d.addBoth(self.shouldRedirect, "/THERE")
2521         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2522         d.addCallback(self.failUnlessNodeKeysAre, [])
2523         return d
2524
2525     def test_POST_mkdir_whendone_queryarg(self):
2526         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2527                       t="mkdir", name="newdir")
2528         d.addBoth(self.shouldRedirect, "/THERE")
2529         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2530         d.addCallback(self.failUnlessNodeKeysAre, [])
2531         return d
2532
2533     def test_POST_bad_t(self):
2534         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2535                              "POST to a directory with bad t=BOGUS",
2536                              self.POST, self.public_url + "/foo", t="BOGUS")
2537         return d
2538
2539     def test_POST_set_children(self, command_name="set_children"):
2540         contents9, n9, newuri9 = self.makefile(9)
2541         contents10, n10, newuri10 = self.makefile(10)
2542         contents11, n11, newuri11 = self.makefile(11)
2543
2544         reqbody = """{
2545                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2546                                                 "size": 0,
2547                                                 "metadata": {
2548                                                   "ctime": 1002777696.7564139,
2549                                                   "mtime": 1002777696.7564139
2550                                                  }
2551                                                } ],
2552                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2553                                                 "size": 1,
2554                                                 "metadata": {
2555                                                   "ctime": 1002777696.7564139,
2556                                                   "mtime": 1002777696.7564139
2557                                                  }
2558                                                } ],
2559                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2560                                                 "size": 2,
2561                                                 "metadata": {
2562                                                   "ctime": 1002777696.7564139,
2563                                                   "mtime": 1002777696.7564139
2564                                                  }
2565                                                } ]
2566                     }""" % (newuri9, newuri10, newuri11)
2567
2568         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2569
2570         d = client.getPage(url, method="POST", postdata=reqbody)
2571         def _then(res):
2572             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2573             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2574             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2575
2576         d.addCallback(_then)
2577         d.addErrback(self.dump_error)
2578         return d
2579
2580     def test_POST_set_children_with_hyphen(self):
2581         return self.test_POST_set_children(command_name="set-children")
2582
2583     def test_POST_link_uri(self):
2584         contents, n, newuri = self.makefile(8)
2585         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2586         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2587         d.addCallback(lambda res:
2588                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2589                                                       contents))
2590         return d
2591
2592     def test_POST_link_uri_replace(self):
2593         contents, n, newuri = self.makefile(8)
2594         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2595         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2596         d.addCallback(lambda res:
2597                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2598                                                       contents))
2599         return d
2600
2601     def test_POST_link_uri_unknown_bad(self):
2602         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2603         d.addBoth(self.shouldFail, error.Error,
2604                   "POST_link_uri_unknown_bad",
2605                   "400 Bad Request",
2606                   "unknown cap in a write slot")
2607         return d
2608
2609     def test_POST_link_uri_unknown_ro_good(self):
2610         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2611         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2612         return d
2613
2614     def test_POST_link_uri_unknown_imm_good(self):
2615         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2616         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2617         return d
2618
2619     def test_POST_link_uri_no_replace_queryarg(self):
2620         contents, n, newuri = self.makefile(8)
2621         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2622                       name="bar.txt", uri=newuri)
2623         d.addBoth(self.shouldFail, error.Error,
2624                   "POST_link_uri_no_replace_queryarg",
2625                   "409 Conflict",
2626                   "There was already a child by that name, and you asked me "
2627                   "to not replace it")
2628         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2629         d.addCallback(self.failUnlessIsBarDotTxt)
2630         return d
2631
2632     def test_POST_link_uri_no_replace_field(self):
2633         contents, n, newuri = self.makefile(8)
2634         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2635                       name="bar.txt", uri=newuri)
2636         d.addBoth(self.shouldFail, error.Error,
2637                   "POST_link_uri_no_replace_field",
2638                   "409 Conflict",
2639                   "There was already a child by that name, and you asked me "
2640                   "to not replace it")
2641         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2642         d.addCallback(self.failUnlessIsBarDotTxt)
2643         return d
2644
2645     def test_POST_delete(self, command_name='delete'):
2646         d = self._foo_node.list()
2647         def _check_before(children):
2648             self.failUnless(u"bar.txt" in children)
2649         d.addCallback(_check_before)
2650         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
2651         d.addCallback(lambda res: self._foo_node.list())
2652         def _check_after(children):
2653             self.failIf(u"bar.txt" in children)
2654         d.addCallback(_check_after)
2655         return d
2656
2657     def test_POST_unlink(self):
2658         return self.test_POST_delete(command_name='unlink')
2659
2660     def test_POST_rename_file(self):
2661         d = self.POST(self.public_url + "/foo", t="rename",
2662                       from_name="bar.txt", to_name='wibble.txt')
2663         d.addCallback(lambda res:
2664                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2665         d.addCallback(lambda res:
2666                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2667         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2668         d.addCallback(self.failUnlessIsBarDotTxt)
2669         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2670         d.addCallback(self.failUnlessIsBarJSON)
2671         return d
2672
2673     def test_POST_rename_file_redundant(self):
2674         d = self.POST(self.public_url + "/foo", t="rename",
2675                       from_name="bar.txt", to_name='bar.txt')
2676         d.addCallback(lambda res:
2677                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2678         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2679         d.addCallback(self.failUnlessIsBarDotTxt)
2680         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2681         d.addCallback(self.failUnlessIsBarJSON)
2682         return d
2683
2684     def test_POST_rename_file_replace(self):
2685         # rename a file and replace a directory with it
2686         d = self.POST(self.public_url + "/foo", t="rename",
2687                       from_name="bar.txt", to_name='empty')
2688         d.addCallback(lambda res:
2689                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2690         d.addCallback(lambda res:
2691                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2692         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2693         d.addCallback(self.failUnlessIsBarDotTxt)
2694         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2695         d.addCallback(self.failUnlessIsBarJSON)
2696         return d
2697
2698     def test_POST_rename_file_no_replace_queryarg(self):
2699         # rename a file and replace a directory with it
2700         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2701                       from_name="bar.txt", to_name='empty')
2702         d.addBoth(self.shouldFail, error.Error,
2703                   "POST_rename_file_no_replace_queryarg",
2704                   "409 Conflict",
2705                   "There was already a child by that name, and you asked me "
2706                   "to not replace it")
2707         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2708         d.addCallback(self.failUnlessIsEmptyJSON)
2709         return d
2710
2711     def test_POST_rename_file_no_replace_field(self):
2712         # rename a file and replace a directory with it
2713         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2714                       from_name="bar.txt", to_name='empty')
2715         d.addBoth(self.shouldFail, error.Error,
2716                   "POST_rename_file_no_replace_field",
2717                   "409 Conflict",
2718                   "There was already a child by that name, and you asked me "
2719                   "to not replace it")
2720         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2721         d.addCallback(self.failUnlessIsEmptyJSON)
2722         return d
2723
2724     def failUnlessIsEmptyJSON(self, res):
2725         data = simplejson.loads(res)
2726         self.failUnlessEqual(data[0], "dirnode", data)
2727         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2728
2729     def test_POST_rename_file_slash_fail(self):
2730         d = self.POST(self.public_url + "/foo", t="rename",
2731                       from_name="bar.txt", to_name='kirk/spock.txt')
2732         d.addBoth(self.shouldFail, error.Error,
2733                   "test_POST_rename_file_slash_fail",
2734                   "400 Bad Request",
2735                   "to_name= may not contain a slash",
2736                   )
2737         d.addCallback(lambda res:
2738                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2739         return d
2740
2741     def test_POST_rename_dir(self):
2742         d = self.POST(self.public_url, t="rename",
2743                       from_name="foo", to_name='plunk')
2744         d.addCallback(lambda res:
2745                       self.failIfNodeHasChild(self.public_root, u"foo"))
2746         d.addCallback(lambda res:
2747                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2748         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2749         d.addCallback(self.failUnlessIsFooJSON)
2750         return d
2751
2752     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2753         """ If target is not None then the redirection has to go to target.  If
2754         statuscode is not None then the redirection has to be accomplished with
2755         that HTTP status code."""
2756         if not isinstance(res, failure.Failure):
2757             to_where = (target is None) and "somewhere" or ("to " + target)
2758             self.fail("%s: we were expecting to get redirected %s, not get an"
2759                       " actual page: %s" % (which, to_where, res))
2760         res.trap(error.PageRedirect)
2761         if statuscode is not None:
2762             self.failUnlessReallyEqual(res.value.status, statuscode,
2763                                        "%s: not a redirect" % which)
2764         if target is not None:
2765             # the PageRedirect does not seem to capture the uri= query arg
2766             # properly, so we can't check for it.
2767             realtarget = self.webish_url + target
2768             self.failUnlessReallyEqual(res.value.location, realtarget,
2769                                        "%s: wrong target" % which)
2770         return res.value.location
2771
2772     def test_GET_URI_form(self):
2773         base = "/uri?uri=%s" % self._bar_txt_uri
2774         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2775         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2776         d = self.GET(base)
2777         d.addBoth(self.shouldRedirect, targetbase)
2778         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2779         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2780         d.addCallback(lambda res: self.GET(base+"&t=json"))
2781         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2782         d.addCallback(self.log, "about to get file by uri")
2783         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2784         d.addCallback(self.failUnlessIsBarDotTxt)
2785         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2786         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2787                                            followRedirect=True))
2788         d.addCallback(self.failUnlessIsFooJSON)
2789         d.addCallback(self.log, "got dir by uri")
2790
2791         return d
2792
2793     def test_GET_URI_form_bad(self):
2794         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2795                              "400 Bad Request", "GET /uri requires uri=",
2796                              self.GET, "/uri")
2797         return d
2798
2799     def test_GET_rename_form(self):
2800         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2801                      followRedirect=True)
2802         def _check(res):
2803             self.failUnless('name="when_done" value="."' in res, res)
2804             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2805         d.addCallback(_check)
2806         return d
2807
2808     def log(self, res, msg):
2809         #print "MSG: %s  RES: %s" % (msg, res)
2810         log.msg(msg)
2811         return res
2812
2813     def test_GET_URI_URL(self):
2814         base = "/uri/%s" % self._bar_txt_uri
2815         d = self.GET(base)
2816         d.addCallback(self.failUnlessIsBarDotTxt)
2817         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2818         d.addCallback(self.failUnlessIsBarDotTxt)
2819         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2820         d.addCallback(self.failUnlessIsBarDotTxt)
2821         return d
2822
2823     def test_GET_URI_URL_dir(self):
2824         base = "/uri/%s?t=json" % self._foo_uri
2825         d = self.GET(base)
2826         d.addCallback(self.failUnlessIsFooJSON)
2827         return d
2828
2829     def test_GET_URI_URL_missing(self):
2830         base = "/uri/%s" % self._bad_file_uri
2831         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2832                                  http.GONE, None, "NotEnoughSharesError",
2833                                  self.GET, base)
2834         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2835         # here? we must arrange for a download to fail after target.open()
2836         # has been called, and then inspect the response to see that it is
2837         # shorter than we expected.
2838         return d
2839
2840     def test_PUT_DIRURL_uri(self):
2841         d = self.s.create_dirnode()
2842         def _made_dir(dn):
2843             new_uri = dn.get_uri()
2844             # replace /foo with a new (empty) directory
2845             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2846             d.addCallback(lambda res:
2847                           self.failUnlessReallyEqual(res.strip(), new_uri))
2848             d.addCallback(lambda res:
2849                           self.failUnlessRWChildURIIs(self.public_root,
2850                                                       u"foo",
2851                                                       new_uri))
2852             return d
2853         d.addCallback(_made_dir)
2854         return d
2855
2856     def test_PUT_DIRURL_uri_noreplace(self):
2857         d = self.s.create_dirnode()
2858         def _made_dir(dn):
2859             new_uri = dn.get_uri()
2860             # replace /foo with a new (empty) directory, but ask that
2861             # replace=false, so it should fail
2862             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2863                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2864                                  self.PUT,
2865                                  self.public_url + "/foo?t=uri&replace=false",
2866                                  new_uri)
2867             d.addCallback(lambda res:
2868                           self.failUnlessRWChildURIIs(self.public_root,
2869                                                       u"foo",
2870                                                       self._foo_uri))
2871             return d
2872         d.addCallback(_made_dir)
2873         return d
2874
2875     def test_PUT_DIRURL_bad_t(self):
2876         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2877                                  "400 Bad Request", "PUT to a directory",
2878                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2879         d.addCallback(lambda res:
2880                       self.failUnlessRWChildURIIs(self.public_root,
2881                                                   u"foo",
2882                                                   self._foo_uri))
2883         return d
2884
2885     def test_PUT_NEWFILEURL_uri(self):
2886         contents, n, new_uri = self.makefile(8)
2887         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2888         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2889         d.addCallback(lambda res:
2890                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2891                                                       contents))
2892         return d
2893
2894     def test_PUT_NEWFILEURL_uri_replace(self):
2895         contents, n, new_uri = self.makefile(8)
2896         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2897         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2898         d.addCallback(lambda res:
2899                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2900                                                       contents))
2901         return d
2902
2903     def test_PUT_NEWFILEURL_uri_no_replace(self):
2904         contents, n, new_uri = self.makefile(8)
2905         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2906         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2907                   "409 Conflict",
2908                   "There was already a child by that name, and you asked me "
2909                   "to not replace it")
2910         return d
2911
2912     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2913         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2914         d.addBoth(self.shouldFail, error.Error,
2915                   "POST_put_uri_unknown_bad",
2916                   "400 Bad Request",
2917                   "unknown cap in a write slot")
2918         return d
2919
2920     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2921         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2922         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2923                       u"put-future-ro.txt")
2924         return d
2925
2926     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2927         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2928         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2929                       u"put-future-imm.txt")
2930         return d
2931
2932     def test_PUT_NEWFILE_URI(self):
2933         file_contents = "New file contents here\n"
2934         d = self.PUT("/uri", file_contents)
2935         def _check(uri):
2936             assert isinstance(uri, str), uri
2937             self.failUnless(uri in FakeCHKFileNode.all_contents)
2938             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2939                                        file_contents)
2940             return self.GET("/uri/%s" % uri)
2941         d.addCallback(_check)
2942         def _check2(res):
2943             self.failUnlessReallyEqual(res, file_contents)
2944         d.addCallback(_check2)
2945         return d
2946
2947     def test_PUT_NEWFILE_URI_not_mutable(self):
2948         file_contents = "New file contents here\n"
2949         d = self.PUT("/uri?mutable=false", file_contents)
2950         def _check(uri):
2951             assert isinstance(uri, str), uri
2952             self.failUnless(uri in FakeCHKFileNode.all_contents)
2953             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2954                                        file_contents)
2955             return self.GET("/uri/%s" % uri)
2956         d.addCallback(_check)
2957         def _check2(res):
2958             self.failUnlessReallyEqual(res, file_contents)
2959         d.addCallback(_check2)
2960         return d
2961
2962     def test_PUT_NEWFILE_URI_only_PUT(self):
2963         d = self.PUT("/uri?t=bogus", "")
2964         d.addBoth(self.shouldFail, error.Error,
2965                   "PUT_NEWFILE_URI_only_PUT",
2966                   "400 Bad Request",
2967                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2968         return d
2969
2970     def test_PUT_NEWFILE_URI_mutable(self):
2971         file_contents = "New file contents here\n"
2972         d = self.PUT("/uri?mutable=true", file_contents)
2973         def _check1(filecap):
2974             filecap = filecap.strip()
2975             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2976             self.filecap = filecap
2977             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2978             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2979             n = self.s.create_node_from_uri(filecap)
2980             return n.download_best_version()
2981         d.addCallback(_check1)
2982         def _check2(data):
2983             self.failUnlessReallyEqual(data, file_contents)
2984             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2985         d.addCallback(_check2)
2986         def _check3(res):
2987             self.failUnlessReallyEqual(res, file_contents)
2988         d.addCallback(_check3)
2989         return d
2990
2991     def test_PUT_mkdir(self):
2992         d = self.PUT("/uri?t=mkdir", "")
2993         def _check(uri):
2994             n = self.s.create_node_from_uri(uri.strip())
2995             d2 = self.failUnlessNodeKeysAre(n, [])
2996             d2.addCallback(lambda res:
2997                            self.GET("/uri/%s?t=json" % uri))
2998             return d2
2999         d.addCallback(_check)
3000         d.addCallback(self.failUnlessIsEmptyJSON)
3001         return d
3002
3003     def test_POST_check(self):
3004         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3005         def _done(res):
3006             # this returns a string form of the results, which are probably
3007             # None since we're using fake filenodes.
3008             # TODO: verify that the check actually happened, by changing
3009             # FakeCHKFileNode to count how many times .check() has been
3010             # called.
3011             pass
3012         d.addCallback(_done)
3013         return d
3014
3015     def test_bad_method(self):
3016         url = self.webish_url + self.public_url + "/foo/bar.txt"
3017         d = self.shouldHTTPError("test_bad_method",
3018                                  501, "Not Implemented",
3019                                  "I don't know how to treat a BOGUS request.",
3020                                  client.getPage, url, method="BOGUS")
3021         return d
3022
3023     def test_short_url(self):
3024         url = self.webish_url + "/uri"
3025         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3026                                  "I don't know how to treat a DELETE request.",
3027                                  client.getPage, url, method="DELETE")
3028         return d
3029
3030     def test_ophandle_bad(self):
3031         url = self.webish_url + "/operations/bogus?t=status"
3032         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3033                                  "unknown/expired handle 'bogus'",
3034                                  client.getPage, url)
3035         return d
3036
3037     def test_ophandle_cancel(self):
3038         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3039                       followRedirect=True)
3040         d.addCallback(lambda ignored:
3041                       self.GET("/operations/128?t=status&output=JSON"))
3042         def _check1(res):
3043             data = simplejson.loads(res)
3044             self.failUnless("finished" in data, res)
3045             monitor = self.ws.root.child_operations.handles["128"][0]
3046             d = self.POST("/operations/128?t=cancel&output=JSON")
3047             def _check2(res):
3048                 data = simplejson.loads(res)
3049                 self.failUnless("finished" in data, res)
3050                 # t=cancel causes the handle to be forgotten
3051                 self.failUnless(monitor.is_cancelled())
3052             d.addCallback(_check2)
3053             return d
3054         d.addCallback(_check1)
3055         d.addCallback(lambda ignored:
3056                       self.shouldHTTPError("test_ophandle_cancel",
3057                                            404, "404 Not Found",
3058                                            "unknown/expired handle '128'",
3059                                            self.GET,
3060                                            "/operations/128?t=status&output=JSON"))
3061         return d
3062
3063     def test_ophandle_retainfor(self):
3064         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3065                       followRedirect=True)
3066         d.addCallback(lambda ignored:
3067                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3068         def _check1(res):
3069             data = simplejson.loads(res)
3070             self.failUnless("finished" in data, res)
3071         d.addCallback(_check1)
3072         # the retain-for=0 will cause the handle to be expired very soon
3073         d.addCallback(lambda ign:
3074             self.clock.advance(2.0))
3075         d.addCallback(lambda ignored:
3076                       self.shouldHTTPError("test_ophandle_retainfor",
3077                                            404, "404 Not Found",
3078                                            "unknown/expired handle '129'",
3079                                            self.GET,
3080                                            "/operations/129?t=status&output=JSON"))
3081         return d
3082
3083     def test_ophandle_release_after_complete(self):
3084         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3085                       followRedirect=True)
3086         d.addCallback(self.wait_for_operation, "130")
3087         d.addCallback(lambda ignored:
3088                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3089         # the release-after-complete=true will cause the handle to be expired
3090         d.addCallback(lambda ignored:
3091                       self.shouldHTTPError("test_ophandle_release_after_complete",
3092                                            404, "404 Not Found",
3093                                            "unknown/expired handle '130'",
3094                                            self.GET,
3095                                            "/operations/130?t=status&output=JSON"))
3096         return d
3097
3098     def test_uncollected_ophandle_expiration(self):
3099         # uncollected ophandles should expire after 4 days
3100         def _make_uncollected_ophandle(ophandle):
3101             d = self.POST(self.public_url +
3102                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3103                           followRedirect=False)
3104             # When we start the operation, the webapi server will want
3105             # to redirect us to the page for the ophandle, so we get
3106             # confirmation that the operation has started. If the
3107             # manifest operation has finished by the time we get there,
3108             # following that redirect (by setting followRedirect=True
3109             # above) has the side effect of collecting the ophandle that
3110             # we've just created, which means that we can't use the
3111             # ophandle to test the uncollected timeout anymore. So,
3112             # instead, catch the 302 here and don't follow it.
3113             d.addBoth(self.should302, "uncollected_ophandle_creation")
3114             return d
3115         # Create an ophandle, don't collect it, then advance the clock by
3116         # 4 days - 1 second and make sure that the ophandle is still there.
3117         d = _make_uncollected_ophandle(131)
3118         d.addCallback(lambda ign:
3119             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3120         d.addCallback(lambda ign:
3121             self.GET("/operations/131?t=status&output=JSON"))
3122         def _check1(res):
3123             data = simplejson.loads(res)
3124             self.failUnless("finished" in data, res)
3125         d.addCallback(_check1)
3126         # Create an ophandle, don't collect it, then try to collect it
3127         # after 4 days. It should be gone.
3128         d.addCallback(lambda ign:
3129             _make_uncollected_ophandle(132))
3130         d.addCallback(lambda ign:
3131             self.clock.advance(96*60*60))
3132         d.addCallback(lambda ign:
3133             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3134                                  404, "404 Not Found",
3135                                  "unknown/expired handle '132'",
3136                                  self.GET,
3137                                  "/operations/132?t=status&output=JSON"))
3138         return d
3139
3140     def test_collected_ophandle_expiration(self):
3141         # collected ophandles should expire after 1 day
3142         def _make_collected_ophandle(ophandle):
3143             d = self.POST(self.public_url +
3144                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3145                           followRedirect=True)
3146             # By following the initial redirect, we collect the ophandle
3147             # we've just created.
3148             return d
3149         # Create a collected ophandle, then collect it after 23 hours
3150         # and 59 seconds to make sure that it is still there.
3151         d = _make_collected_ophandle(133)
3152         d.addCallback(lambda ign:
3153             self.clock.advance((24*60*60) - 1))
3154         d.addCallback(lambda ign:
3155             self.GET("/operations/133?t=status&output=JSON"))
3156         def _check1(res):
3157             data = simplejson.loads(res)
3158             self.failUnless("finished" in data, res)
3159         d.addCallback(_check1)
3160         # Create another uncollected ophandle, then try to collect it
3161         # after 24 hours to make sure that it is gone.
3162         d.addCallback(lambda ign:
3163             _make_collected_ophandle(134))
3164         d.addCallback(lambda ign:
3165             self.clock.advance(24*60*60))
3166         d.addCallback(lambda ign:
3167             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3168                                  404, "404 Not Found",
3169                                  "unknown/expired handle '134'",
3170                                  self.GET,
3171                                  "/operations/134?t=status&output=JSON"))
3172         return d
3173
3174     def test_incident(self):
3175         d = self.POST("/report_incident", details="eek")
3176         def _done(res):
3177             self.failUnless("Thank you for your report!" in res, res)
3178         d.addCallback(_done)
3179         return d
3180
3181     def test_static(self):
3182         webdir = os.path.join(self.staticdir, "subdir")
3183         fileutil.make_dirs(webdir)
3184         f = open(os.path.join(webdir, "hello.txt"), "wb")
3185         f.write("hello")
3186         f.close()
3187
3188         d = self.GET("/static/subdir/hello.txt")
3189         def _check(res):
3190             self.failUnlessReallyEqual(res, "hello")
3191         d.addCallback(_check)
3192         return d
3193
3194
3195 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3196     def test_load_file(self):
3197         # This will raise an exception unless a well-formed XML file is found under that name.
3198         common.getxmlfile('directory.xhtml').load()
3199
3200     def test_parse_replace_arg(self):
3201         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3202         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3203         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3204                                    "only-files")
3205         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3206                         common.parse_replace_arg, "only_fles")
3207
3208     def test_abbreviate_time(self):
3209         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3210         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3211         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3212         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3213         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3214         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3215
3216     def test_compute_rate(self):
3217         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3218         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3219         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3220         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3221         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3222         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3223         self.shouldFail(AssertionError, "test_compute_rate", "",
3224                         common.compute_rate, -100, 10)
3225         self.shouldFail(AssertionError, "test_compute_rate", "",
3226                         common.compute_rate, 100, -10)
3227
3228         # Sanity check
3229         rate = common.compute_rate(10*1000*1000, 1)
3230         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3231
3232     def test_abbreviate_rate(self):
3233         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3234         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3235         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3236         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3237
3238     def test_abbreviate_size(self):
3239         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3240         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3241         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3242         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3243         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3244
3245     def test_plural(self):
3246         def convert(s):
3247             return "%d second%s" % (s, status.plural(s))
3248         self.failUnlessReallyEqual(convert(0), "0 seconds")
3249         self.failUnlessReallyEqual(convert(1), "1 second")
3250         self.failUnlessReallyEqual(convert(2), "2 seconds")
3251         def convert2(s):
3252             return "has share%s: %s" % (status.plural(s), ",".join(s))
3253         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3254         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3255         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3256
3257
3258 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3259
3260     def CHECK(self, ign, which, args, clientnum=0):
3261         fileurl = self.fileurls[which]
3262         url = fileurl + "?" + args
3263         return self.GET(url, method="POST", clientnum=clientnum)
3264
3265     def test_filecheck(self):
3266         self.basedir = "web/Grid/filecheck"
3267         self.set_up_grid()
3268         c0 = self.g.clients[0]
3269         self.uris = {}
3270         DATA = "data" * 100
3271         d = c0.upload(upload.Data(DATA, convergence=""))
3272         def _stash_uri(ur, which):
3273             self.uris[which] = ur.uri
3274         d.addCallback(_stash_uri, "good")
3275         d.addCallback(lambda ign:
3276                       c0.upload(upload.Data(DATA+"1", convergence="")))
3277         d.addCallback(_stash_uri, "sick")
3278         d.addCallback(lambda ign:
3279                       c0.upload(upload.Data(DATA+"2", convergence="")))
3280         d.addCallback(_stash_uri, "dead")
3281         def _stash_mutable_uri(n, which):
3282             self.uris[which] = n.get_uri()
3283             assert isinstance(self.uris[which], str)
3284         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3285         d.addCallback(_stash_mutable_uri, "corrupt")
3286         d.addCallback(lambda ign:
3287                       c0.upload(upload.Data("literal", convergence="")))
3288         d.addCallback(_stash_uri, "small")
3289         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3290         d.addCallback(_stash_mutable_uri, "smalldir")
3291
3292         def _compute_fileurls(ignored):
3293             self.fileurls = {}
3294             for which in self.uris:
3295                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3296         d.addCallback(_compute_fileurls)
3297
3298         def _clobber_shares(ignored):
3299             good_shares = self.find_uri_shares(self.uris["good"])
3300             self.failUnlessReallyEqual(len(good_shares), 10)
3301             sick_shares = self.find_uri_shares(self.uris["sick"])
3302             os.unlink(sick_shares[0][2])
3303             dead_shares = self.find_uri_shares(self.uris["dead"])
3304             for i in range(1, 10):
3305                 os.unlink(dead_shares[i][2])
3306             c_shares = self.find_uri_shares(self.uris["corrupt"])
3307             cso = CorruptShareOptions()
3308             cso.stdout = StringIO()
3309             cso.parseOptions([c_shares[0][2]])
3310             corrupt_share(cso)
3311         d.addCallback(_clobber_shares)
3312
3313         d.addCallback(self.CHECK, "good", "t=check")
3314         def _got_html_good(res):
3315             self.failUnless("Healthy" in res, res)
3316             self.failIf("Not Healthy" in res, res)
3317         d.addCallback(_got_html_good)
3318         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3319         def _got_html_good_return_to(res):
3320             self.failUnless("Healthy" in res, res)
3321             self.failIf("Not Healthy" in res, res)
3322             self.failUnless('<a href="somewhere">Return to file'
3323                             in res, res)
3324         d.addCallback(_got_html_good_return_to)
3325         d.addCallback(self.CHECK, "good", "t=check&output=json")
3326         def _got_json_good(res):
3327             r = simplejson.loads(res)
3328             self.failUnlessEqual(r["summary"], "Healthy")
3329             self.failUnless(r["results"]["healthy"])
3330             self.failIf(r["results"]["needs-rebalancing"])
3331             self.failUnless(r["results"]["recoverable"])
3332         d.addCallback(_got_json_good)
3333
3334         d.addCallback(self.CHECK, "small", "t=check")
3335         def _got_html_small(res):
3336             self.failUnless("Literal files are always healthy" in res, res)
3337             self.failIf("Not Healthy" in res, res)
3338         d.addCallback(_got_html_small)
3339         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3340         def _got_html_small_return_to(res):
3341             self.failUnless("Literal files are always healthy" in res, res)
3342             self.failIf("Not Healthy" in res, res)
3343             self.failUnless('<a href="somewhere">Return to file'
3344                             in res, res)
3345         d.addCallback(_got_html_small_return_to)
3346         d.addCallback(self.CHECK, "small", "t=check&output=json")
3347         def _got_json_small(res):
3348             r = simplejson.loads(res)
3349             self.failUnlessEqual(r["storage-index"], "")
3350             self.failUnless(r["results"]["healthy"])
3351         d.addCallback(_got_json_small)
3352
3353         d.addCallback(self.CHECK, "smalldir", "t=check")
3354         def _got_html_smalldir(res):
3355             self.failUnless("Literal files are always healthy" in res, res)
3356             self.failIf("Not Healthy" in res, res)
3357         d.addCallback(_got_html_smalldir)
3358         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3359         def _got_json_smalldir(res):
3360             r = simplejson.loads(res)
3361             self.failUnlessEqual(r["storage-index"], "")
3362             self.failUnless(r["results"]["healthy"])
3363         d.addCallback(_got_json_smalldir)
3364
3365         d.addCallback(self.CHECK, "sick", "t=check")
3366         def _got_html_sick(res):
3367             self.failUnless("Not Healthy" in res, res)
3368         d.addCallback(_got_html_sick)
3369         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3370         def _got_json_sick(res):
3371             r = simplejson.loads(res)
3372             self.failUnlessEqual(r["summary"],
3373                                  "Not Healthy: 9 shares (enc 3-of-10)")
3374             self.failIf(r["results"]["healthy"])
3375             self.failIf(r["results"]["needs-rebalancing"])
3376             self.failUnless(r["results"]["recoverable"])
3377         d.addCallback(_got_json_sick)
3378
3379         d.addCallback(self.CHECK, "dead", "t=check")
3380         def _got_html_dead(res):
3381             self.failUnless("Not Healthy" in res, res)
3382         d.addCallback(_got_html_dead)
3383         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3384         def _got_json_dead(res):
3385             r = simplejson.loads(res)
3386             self.failUnlessEqual(r["summary"],
3387                                  "Not Healthy: 1 shares (enc 3-of-10)")
3388             self.failIf(r["results"]["healthy"])
3389             self.failIf(r["results"]["needs-rebalancing"])
3390             self.failIf(r["results"]["recoverable"])
3391         d.addCallback(_got_json_dead)
3392
3393         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3394         def _got_html_corrupt(res):
3395             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3396         d.addCallback(_got_html_corrupt)
3397         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3398         def _got_json_corrupt(res):
3399             r = simplejson.loads(res)
3400             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3401                             r["summary"])
3402             self.failIf(r["results"]["healthy"])
3403             self.failUnless(r["results"]["recoverable"])
3404             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3405             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3406         d.addCallback(_got_json_corrupt)
3407
3408         d.addErrback(self.explain_web_error)
3409         return d
3410
3411     def test_repair_html(self):
3412         self.basedir = "web/Grid/repair_html"
3413         self.set_up_grid()
3414         c0 = self.g.clients[0]
3415         self.uris = {}
3416         DATA = "data" * 100
3417         d = c0.upload(upload.Data(DATA, convergence=""))
3418         def _stash_uri(ur, which):
3419             self.uris[which] = ur.uri
3420         d.addCallback(_stash_uri, "good")
3421         d.addCallback(lambda ign:
3422                       c0.upload(upload.Data(DATA+"1", convergence="")))
3423         d.addCallback(_stash_uri, "sick")
3424         d.addCallback(lambda ign:
3425                       c0.upload(upload.Data(DATA+"2", convergence="")))
3426         d.addCallback(_stash_uri, "dead")
3427         def _stash_mutable_uri(n, which):
3428             self.uris[which] = n.get_uri()
3429             assert isinstance(self.uris[which], str)
3430         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3431         d.addCallback(_stash_mutable_uri, "corrupt")
3432
3433         def _compute_fileurls(ignored):
3434             self.fileurls = {}
3435             for which in self.uris:
3436                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3437         d.addCallback(_compute_fileurls)
3438
3439         def _clobber_shares(ignored):
3440             good_shares = self.find_uri_shares(self.uris["good"])
3441             self.failUnlessReallyEqual(len(good_shares), 10)
3442             sick_shares = self.find_uri_shares(self.uris["sick"])
3443             os.unlink(sick_shares[0][2])
3444             dead_shares = self.find_uri_shares(self.uris["dead"])
3445             for i in range(1, 10):
3446                 os.unlink(dead_shares[i][2])
3447             c_shares = self.find_uri_shares(self.uris["corrupt"])
3448             cso = CorruptShareOptions()
3449             cso.stdout = StringIO()
3450             cso.parseOptions([c_shares[0][2]])
3451             corrupt_share(cso)
3452         d.addCallback(_clobber_shares)
3453
3454         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3455         def _got_html_good(res):
3456             self.failUnless("Healthy" in res, res)
3457             self.failIf("Not Healthy" in res, res)
3458             self.failUnless("No repair necessary" in res, res)
3459         d.addCallback(_got_html_good)
3460
3461         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3462         def _got_html_sick(res):
3463             self.failUnless("Healthy : healthy" in res, res)
3464             self.failIf("Not Healthy" in res, res)
3465             self.failUnless("Repair successful" in res, res)
3466         d.addCallback(_got_html_sick)
3467
3468         # repair of a dead file will fail, of course, but it isn't yet
3469         # clear how this should be reported. Right now it shows up as
3470         # a "410 Gone".
3471         #
3472         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3473         #def _got_html_dead(res):
3474         #    print res
3475         #    self.failUnless("Healthy : healthy" in res, res)
3476         #    self.failIf("Not Healthy" in res, res)
3477         #    self.failUnless("No repair necessary" in res, res)
3478         #d.addCallback(_got_html_dead)
3479
3480         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3481         def _got_html_corrupt(res):
3482             self.failUnless("Healthy : Healthy" in res, res)
3483             self.failIf("Not Healthy" in res, res)
3484             self.failUnless("Repair successful" in res, res)
3485         d.addCallback(_got_html_corrupt)
3486
3487         d.addErrback(self.explain_web_error)
3488         return d
3489
3490     def test_repair_json(self):
3491         self.basedir = "web/Grid/repair_json"
3492         self.set_up_grid()
3493         c0 = self.g.clients[0]
3494         self.uris = {}
3495         DATA = "data" * 100
3496         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3497         def _stash_uri(ur, which):
3498             self.uris[which] = ur.uri
3499         d.addCallback(_stash_uri, "sick")
3500
3501         def _compute_fileurls(ignored):
3502             self.fileurls = {}
3503             for which in self.uris:
3504                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3505         d.addCallback(_compute_fileurls)
3506
3507         def _clobber_shares(ignored):
3508             sick_shares = self.find_uri_shares(self.uris["sick"])
3509             os.unlink(sick_shares[0][2])
3510         d.addCallback(_clobber_shares)
3511
3512         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3513         def _got_json_sick(res):
3514             r = simplejson.loads(res)
3515             self.failUnlessReallyEqual(r["repair-attempted"], True)
3516             self.failUnlessReallyEqual(r["repair-successful"], True)
3517             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3518                                  "Not Healthy: 9 shares (enc 3-of-10)")
3519             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3520             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3521             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3522         d.addCallback(_got_json_sick)
3523
3524         d.addErrback(self.explain_web_error)
3525         return d
3526
3527     def test_unknown(self, immutable=False):
3528         self.basedir = "web/Grid/unknown"
3529         if immutable:
3530             self.basedir = "web/Grid/unknown-immutable"
3531
3532         self.set_up_grid()
3533         c0 = self.g.clients[0]
3534         self.uris = {}
3535         self.fileurls = {}
3536
3537         # the future cap format may contain slashes, which must be tolerated
3538         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3539                                                            safe="")
3540
3541         if immutable:
3542             name = u"future-imm"
3543             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3544             d = c0.create_immutable_dirnode({name: (future_node, {})})
3545         else:
3546             name = u"future"
3547             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3548             d = c0.create_dirnode()
3549
3550         def _stash_root_and_create_file(n):
3551             self.rootnode = n
3552             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3553             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3554             if not immutable:
3555                 return self.rootnode.set_node(name, future_node)
3556         d.addCallback(_stash_root_and_create_file)
3557
3558         # make sure directory listing tolerates unknown nodes
3559         d.addCallback(lambda ign: self.GET(self.rooturl))
3560         def _check_directory_html(res, expected_type_suffix):
3561             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3562                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3563                                  re.DOTALL)
3564             self.failUnless(re.search(pattern, res), res)
3565             # find the More Info link for name, should be relative
3566             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3567             info_url = mo.group(1)
3568             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3569         if immutable:
3570             d.addCallback(_check_directory_html, "-IMM")
3571         else:
3572             d.addCallback(_check_directory_html, "")
3573
3574         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3575         def _check_directory_json(res, expect_rw_uri):
3576             data = simplejson.loads(res)
3577             self.failUnlessEqual(data[0], "dirnode")
3578             f = data[1]["children"][name]
3579             self.failUnlessEqual(f[0], "unknown")
3580             if expect_rw_uri:
3581                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
3582             else:
3583                 self.failIfIn("rw_uri", f[1])
3584             if immutable:
3585                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
3586             else:
3587                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
3588             self.failUnless("metadata" in f[1])
3589         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3590
3591         def _check_info(res, expect_rw_uri, expect_ro_uri):
3592             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3593             if expect_rw_uri:
3594                 self.failUnlessIn(unknown_rwcap, res)
3595             if expect_ro_uri:
3596                 if immutable:
3597                     self.failUnlessIn(unknown_immcap, res)
3598                 else:
3599                     self.failUnlessIn(unknown_rocap, res)
3600             else:
3601                 self.failIfIn(unknown_rocap, res)
3602             self.failIfIn("Raw data as", res)
3603             self.failIfIn("Directory writecap", res)
3604             self.failIfIn("Checker Operations", res)
3605             self.failIfIn("Mutable File Operations", res)
3606             self.failIfIn("Directory Operations", res)
3607
3608         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3609         # why they fail. Possibly related to ticket #922.
3610
3611         d.addCallback(lambda ign: self.GET(expected_info_url))
3612         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3613         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3614         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3615
3616         def _check_json(res, expect_rw_uri):
3617             data = simplejson.loads(res)
3618             self.failUnlessEqual(data[0], "unknown")
3619             if expect_rw_uri:
3620                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
3621             else:
3622                 self.failIfIn("rw_uri", data[1])
3623
3624             if immutable:
3625                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
3626                 self.failUnlessReallyEqual(data[1]["mutable"], False)
3627             elif expect_rw_uri:
3628                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3629                 self.failUnlessReallyEqual(data[1]["mutable"], True)
3630             else:
3631                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
3632                 self.failIf("mutable" in data[1], data[1])
3633
3634             # TODO: check metadata contents
3635             self.failUnless("metadata" in data[1])
3636
3637         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3638         d.addCallback(_check_json, expect_rw_uri=not immutable)
3639
3640         # and make sure that a read-only version of the directory can be
3641         # rendered too. This version will not have unknown_rwcap, whether
3642         # or not future_node was immutable.
3643         d.addCallback(lambda ign: self.GET(self.rourl))
3644         if immutable:
3645             d.addCallback(_check_directory_html, "-IMM")
3646         else:
3647             d.addCallback(_check_directory_html, "-RO")
3648
3649         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3650         d.addCallback(_check_directory_json, expect_rw_uri=False)
3651
3652         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3653         d.addCallback(_check_json, expect_rw_uri=False)
3654
3655         # TODO: check that getting t=info from the Info link in the ro directory
3656         # works, and does not include the writecap URI.
3657         return d
3658
3659     def test_immutable_unknown(self):
3660         return self.test_unknown(immutable=True)
3661
3662     def test_mutant_dirnodes_are_omitted(self):
3663         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3664
3665         self.set_up_grid()
3666         c = self.g.clients[0]
3667         nm = c.nodemaker
3668         self.uris = {}
3669         self.fileurls = {}
3670
3671         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3672         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3673         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3674
3675         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3676         # test the dirnode and web layers separately.
3677
3678         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3679         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3680         # When the directory is read, the mutants should be silently disposed of, leaving
3681         # their lonely sibling.
3682         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3683         # because immutable directories don't have a writecap and therefore that field
3684         # isn't (and can't be) decrypted.
3685         # TODO: The field still exists in the netstring. Technically we should check what
3686         # happens if something is put there (_unpack_contents should raise ValueError),
3687         # but that can wait.
3688
3689         lonely_child = nm.create_from_cap(lonely_uri)
3690         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3691         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3692
3693         def _by_hook_or_by_crook():
3694             return True
3695         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3696             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3697
3698         mutant_write_in_ro_child.get_write_uri    = lambda: None
3699         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3700
3701         kids = {u"lonely":      (lonely_child, {}),
3702                 u"ro":          (mutant_ro_child, {}),
3703                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3704                 }
3705         d = c.create_immutable_dirnode(kids)
3706
3707         def _created(dn):
3708             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3709             self.failIf(dn.is_mutable())
3710             self.failUnless(dn.is_readonly())
3711             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3712             self.failIf(hasattr(dn._node, 'get_writekey'))
3713             rep = str(dn)
3714             self.failUnless("RO-IMM" in rep)
3715             cap = dn.get_cap()
3716             self.failUnlessIn("CHK", cap.to_string())
3717             self.cap = cap
3718             self.rootnode = dn
3719             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3720             return download_to_data(dn._node)
3721         d.addCallback(_created)
3722
3723         def _check_data(data):
3724             # Decode the netstring representation of the directory to check that all children
3725             # are present. This is a bit of an abstraction violation, but there's not really
3726             # any other way to do it given that the real DirectoryNode._unpack_contents would
3727             # strip the mutant children out (which is what we're trying to test, later).
3728             position = 0
3729             numkids = 0
3730             while position < len(data):
3731                 entries, position = split_netstring(data, 1, position)
3732                 entry = entries[0]
3733                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3734                 name = name_utf8.decode("utf-8")
3735                 self.failUnless(rwcapdata == "")
3736                 self.failUnless(name in kids)
3737                 (expected_child, ign) = kids[name]
3738                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3739                 numkids += 1
3740
3741             self.failUnlessReallyEqual(numkids, 3)
3742             return self.rootnode.list()
3743         d.addCallback(_check_data)
3744
3745         # Now when we use the real directory listing code, the mutants should be absent.
3746         def _check_kids(children):
3747             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3748             lonely_node, lonely_metadata = children[u"lonely"]
3749
3750             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3751             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3752         d.addCallback(_check_kids)
3753
3754         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3755         d.addCallback(lambda n: n.list())
3756         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3757
3758         # Make sure the lonely child can be listed in HTML...
3759         d.addCallback(lambda ign: self.GET(self.rooturl))
3760         def _check_html(res):
3761             self.failIfIn("URI:SSK", res)
3762             get_lonely = "".join([r'<td>FILE</td>',
3763                                   r'\s+<td>',
3764                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3765                                   r'</td>',
3766                                   r'\s+<td align="right">%d</td>' % len("one"),
3767                                   ])
3768             self.failUnless(re.search(get_lonely, res), res)
3769
3770             # find the More Info link for name, should be relative
3771             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3772             info_url = mo.group(1)
3773             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3774         d.addCallback(_check_html)
3775
3776         # ... and in JSON.
3777         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3778         def _check_json(res):
3779             data = simplejson.loads(res)
3780             self.failUnlessEqual(data[0], "dirnode")
3781             listed_children = data[1]["children"]
3782             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3783             ll_type, ll_data = listed_children[u"lonely"]
3784             self.failUnlessEqual(ll_type, "filenode")
3785             self.failIf("rw_uri" in ll_data)
3786             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
3787         d.addCallback(_check_json)
3788         return d
3789
3790     def test_deep_check(self):
3791         self.basedir = "web/Grid/deep_check"
3792         self.set_up_grid()
3793         c0 = self.g.clients[0]
3794         self.uris = {}
3795         self.fileurls = {}
3796         DATA = "data" * 100
3797         d = c0.create_dirnode()
3798         def _stash_root_and_create_file(n):
3799             self.rootnode = n
3800             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3801             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3802         d.addCallback(_stash_root_and_create_file)
3803         def _stash_uri(fn, which):
3804             self.uris[which] = fn.get_uri()
3805             return fn
3806         d.addCallback(_stash_uri, "good")
3807         d.addCallback(lambda ign:
3808                       self.rootnode.add_file(u"small",
3809                                              upload.Data("literal",
3810                                                         convergence="")))
3811         d.addCallback(_stash_uri, "small")
3812         d.addCallback(lambda ign:
3813                       self.rootnode.add_file(u"sick",
3814                                              upload.Data(DATA+"1",
3815                                                         convergence="")))
3816         d.addCallback(_stash_uri, "sick")
3817
3818         # this tests that deep-check and stream-manifest will ignore
3819         # UnknownNode instances. Hopefully this will also cover deep-stats.
3820         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3821         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3822
3823         def _clobber_shares(ignored):
3824             self.delete_shares_numbered(self.uris["sick"], [0,1])
3825         d.addCallback(_clobber_shares)
3826
3827         # root
3828         # root/good
3829         # root/small
3830         # root/sick
3831         # root/future
3832
3833         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3834         def _done(res):
3835             try:
3836                 units = [simplejson.loads(line)
3837                          for line in res.splitlines()
3838                          if line]
3839             except ValueError:
3840                 print "response is:", res
3841                 print "undecodeable line was '%s'" % line
3842                 raise
3843             self.failUnlessReallyEqual(len(units), 5+1)
3844             # should be parent-first
3845             u0 = units[0]
3846             self.failUnlessEqual(u0["path"], [])
3847             self.failUnlessEqual(u0["type"], "directory")
3848             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
3849             u0cr = u0["check-results"]
3850             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3851
3852             ugood = [u for u in units
3853                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3854             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
3855             ugoodcr = ugood["check-results"]
3856             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3857
3858             stats = units[-1]
3859             self.failUnlessEqual(stats["type"], "stats")
3860             s = stats["stats"]
3861             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3862             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3863             self.failUnlessReallyEqual(s["count-directories"], 1)
3864             self.failUnlessReallyEqual(s["count-unknown"], 1)
3865         d.addCallback(_done)
3866
3867         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3868         def _check_manifest(res):
3869             self.failUnless(res.endswith("\n"))
3870             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3871             self.failUnlessReallyEqual(len(units), 5+1)
3872             self.failUnlessEqual(units[-1]["type"], "stats")
3873             first = units[0]
3874             self.failUnlessEqual(first["path"], [])
3875             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
3876             self.failUnlessEqual(first["type"], "directory")
3877             stats = units[-1]["stats"]
3878             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3879             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3880             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3881             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3882             self.failUnlessReallyEqual(stats["count-unknown"], 1)
3883         d.addCallback(_check_manifest)
3884
3885         # now add root/subdir and root/subdir/grandchild, then make subdir
3886         # unrecoverable, then see what happens
3887
3888         d.addCallback(lambda ign:
3889                       self.rootnode.create_subdirectory(u"subdir"))
3890         d.addCallback(_stash_uri, "subdir")
3891         d.addCallback(lambda subdir_node:
3892                       subdir_node.add_file(u"grandchild",
3893                                            upload.Data(DATA+"2",
3894                                                        convergence="")))
3895         d.addCallback(_stash_uri, "grandchild")
3896
3897         d.addCallback(lambda ign:
3898                       self.delete_shares_numbered(self.uris["subdir"],
3899                                                   range(1, 10)))
3900
3901         # root
3902         # root/good
3903         # root/small
3904         # root/sick
3905         # root/future
3906         # root/subdir [unrecoverable]
3907         # root/subdir/grandchild
3908
3909         # how should a streaming-JSON API indicate fatal error?
3910         # answer: emit ERROR: instead of a JSON string
3911
3912         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3913         def _check_broken_manifest(res):
3914             lines = res.splitlines()
3915             error_lines = [i
3916                            for (i,line) in enumerate(lines)
3917                            if line.startswith("ERROR:")]
3918             if not error_lines:
3919                 self.fail("no ERROR: in output: %s" % (res,))
3920             first_error = error_lines[0]
3921             error_line = lines[first_error]
3922             error_msg = lines[first_error+1:]
3923             error_msg_s = "\n".join(error_msg) + "\n"
3924             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3925                               error_line)
3926             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3927             units = [simplejson.loads(line) for line in lines[:first_error]]
3928             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3929             last_unit = units[-1]
3930             self.failUnlessEqual(last_unit["path"], ["subdir"])
3931         d.addCallback(_check_broken_manifest)
3932
3933         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3934         def _check_broken_deepcheck(res):
3935             lines = res.splitlines()
3936             error_lines = [i
3937                            for (i,line) in enumerate(lines)
3938                            if line.startswith("ERROR:")]
3939             if not error_lines:
3940                 self.fail("no ERROR: in output: %s" % (res,))
3941             first_error = error_lines[0]
3942             error_line = lines[first_error]
3943             error_msg = lines[first_error+1:]
3944             error_msg_s = "\n".join(error_msg) + "\n"
3945             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3946                               error_line)
3947             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3948             units = [simplejson.loads(line) for line in lines[:first_error]]
3949             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3950             last_unit = units[-1]
3951             self.failUnlessEqual(last_unit["path"], ["subdir"])
3952             r = last_unit["check-results"]["results"]
3953             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3954             self.failUnlessReallyEqual(r["count-shares-good"], 1)
3955             self.failUnlessReallyEqual(r["recoverable"], False)
3956         d.addCallback(_check_broken_deepcheck)
3957
3958         d.addErrback(self.explain_web_error)
3959         return d
3960
3961     def test_deep_check_and_repair(self):
3962         self.basedir = "web/Grid/deep_check_and_repair"
3963         self.set_up_grid()
3964         c0 = self.g.clients[0]
3965         self.uris = {}
3966         self.fileurls = {}
3967         DATA = "data" * 100
3968         d = c0.create_dirnode()
3969         def _stash_root_and_create_file(n):
3970             self.rootnode = n
3971             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3972             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3973         d.addCallback(_stash_root_and_create_file)
3974         def _stash_uri(fn, which):
3975             self.uris[which] = fn.get_uri()
3976         d.addCallback(_stash_uri, "good")
3977         d.addCallback(lambda ign:
3978                       self.rootnode.add_file(u"small",
3979                                              upload.Data("literal",
3980                                                         convergence="")))
3981         d.addCallback(_stash_uri, "small")
3982         d.addCallback(lambda ign:
3983                       self.rootnode.add_file(u"sick",
3984                                              upload.Data(DATA+"1",
3985                                                         convergence="")))
3986         d.addCallback(_stash_uri, "sick")
3987         #d.addCallback(lambda ign:
3988         #              self.rootnode.add_file(u"dead",
3989         #                                     upload.Data(DATA+"2",
3990         #                                                convergence="")))
3991         #d.addCallback(_stash_uri, "dead")
3992
3993         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3994         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3995         #d.addCallback(_stash_uri, "corrupt")
3996
3997         def _clobber_shares(ignored):
3998             good_shares = self.find_uri_shares(self.uris["good"])
3999             self.failUnlessReallyEqual(len(good_shares), 10)
4000             sick_shares = self.find_uri_shares(self.uris["sick"])
4001             os.unlink(sick_shares[0][2])
4002             #dead_shares = self.find_uri_shares(self.uris["dead"])
4003             #for i in range(1, 10):
4004             #    os.unlink(dead_shares[i][2])
4005
4006             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4007             #cso = CorruptShareOptions()
4008             #cso.stdout = StringIO()
4009             #cso.parseOptions([c_shares[0][2]])
4010             #corrupt_share(cso)
4011         d.addCallback(_clobber_shares)
4012
4013         # root
4014         # root/good   CHK, 10 shares
4015         # root/small  LIT
4016         # root/sick   CHK, 9 shares
4017
4018         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4019         def _done(res):
4020             units = [simplejson.loads(line)
4021                      for line in res.splitlines()
4022                      if line]
4023             self.failUnlessReallyEqual(len(units), 4+1)
4024             # should be parent-first
4025             u0 = units[0]
4026             self.failUnlessEqual(u0["path"], [])
4027             self.failUnlessEqual(u0["type"], "directory")
4028             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4029             u0crr = u0["check-and-repair-results"]
4030             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4031             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4032
4033             ugood = [u for u in units
4034                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4035             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4036             ugoodcrr = ugood["check-and-repair-results"]
4037             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4038             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4039
4040             usick = [u for u in units
4041                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4042             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4043             usickcrr = usick["check-and-repair-results"]
4044             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4045             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4046             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4047             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4048
4049             stats = units[-1]
4050             self.failUnlessEqual(stats["type"], "stats")
4051             s = stats["stats"]
4052             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4053             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4054             self.failUnlessReallyEqual(s["count-directories"], 1)
4055         d.addCallback(_done)
4056
4057         d.addErrback(self.explain_web_error)
4058         return d
4059
4060     def _count_leases(self, ignored, which):
4061         u = self.uris[which]
4062         shares = self.find_uri_shares(u)
4063         lease_counts = []
4064         for shnum, serverid, fn in shares:
4065             sf = get_share_file(fn)
4066             num_leases = len(list(sf.get_leases()))
4067         lease_counts.append( (fn, num_leases) )
4068         return lease_counts
4069
4070     def _assert_leasecount(self, lease_counts, expected):
4071         for (fn, num_leases) in lease_counts:
4072             if num_leases != expected:
4073                 self.fail("expected %d leases, have %d, on %s" %
4074                           (expected, num_leases, fn))
4075
4076     def test_add_lease(self):
4077         self.basedir = "web/Grid/add_lease"
4078         self.set_up_grid(num_clients=2)
4079         c0 = self.g.clients[0]
4080         self.uris = {}
4081         DATA = "data" * 100
4082         d = c0.upload(upload.Data(DATA, convergence=""))
4083         def _stash_uri(ur, which):
4084             self.uris[which] = ur.uri
4085         d.addCallback(_stash_uri, "one")
4086         d.addCallback(lambda ign:
4087                       c0.upload(upload.Data(DATA+"1", convergence="")))
4088         d.addCallback(_stash_uri, "two")
4089         def _stash_mutable_uri(n, which):
4090             self.uris[which] = n.get_uri()
4091             assert isinstance(self.uris[which], str)
4092         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
4093         d.addCallback(_stash_mutable_uri, "mutable")
4094
4095         def _compute_fileurls(ignored):
4096             self.fileurls = {}
4097             for which in self.uris:
4098                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4099         d.addCallback(_compute_fileurls)
4100
4101         d.addCallback(self._count_leases, "one")
4102         d.addCallback(self._assert_leasecount, 1)
4103         d.addCallback(self._count_leases, "two")
4104         d.addCallback(self._assert_leasecount, 1)
4105         d.addCallback(self._count_leases, "mutable")
4106         d.addCallback(self._assert_leasecount, 1)
4107
4108         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4109         def _got_html_good(res):
4110             self.failUnless("Healthy" in res, res)
4111             self.failIf("Not Healthy" in res, res)
4112         d.addCallback(_got_html_good)
4113
4114         d.addCallback(self._count_leases, "one")
4115         d.addCallback(self._assert_leasecount, 1)
4116         d.addCallback(self._count_leases, "two")
4117         d.addCallback(self._assert_leasecount, 1)
4118         d.addCallback(self._count_leases, "mutable")
4119         d.addCallback(self._assert_leasecount, 1)
4120
4121         # this CHECK uses the original client, which uses the same
4122         # lease-secrets, so it will just renew the original lease
4123         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4124         d.addCallback(_got_html_good)
4125
4126         d.addCallback(self._count_leases, "one")
4127         d.addCallback(self._assert_leasecount, 1)
4128         d.addCallback(self._count_leases, "two")
4129         d.addCallback(self._assert_leasecount, 1)
4130         d.addCallback(self._count_leases, "mutable")
4131         d.addCallback(self._assert_leasecount, 1)
4132
4133         # this CHECK uses an alternate client, which adds a second lease
4134         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4135         d.addCallback(_got_html_good)
4136
4137         d.addCallback(self._count_leases, "one")
4138         d.addCallback(self._assert_leasecount, 2)
4139         d.addCallback(self._count_leases, "two")
4140         d.addCallback(self._assert_leasecount, 1)
4141         d.addCallback(self._count_leases, "mutable")
4142         d.addCallback(self._assert_leasecount, 1)
4143
4144         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4145         d.addCallback(_got_html_good)
4146
4147         d.addCallback(self._count_leases, "one")
4148         d.addCallback(self._assert_leasecount, 2)
4149         d.addCallback(self._count_leases, "two")
4150         d.addCallback(self._assert_leasecount, 1)
4151         d.addCallback(self._count_leases, "mutable")
4152         d.addCallback(self._assert_leasecount, 1)
4153
4154         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4155                       clientnum=1)
4156         d.addCallback(_got_html_good)
4157
4158         d.addCallback(self._count_leases, "one")
4159         d.addCallback(self._assert_leasecount, 2)
4160         d.addCallback(self._count_leases, "two")
4161         d.addCallback(self._assert_leasecount, 1)
4162         d.addCallback(self._count_leases, "mutable")
4163         d.addCallback(self._assert_leasecount, 2)
4164
4165         d.addErrback(self.explain_web_error)
4166         return d
4167
4168     def test_deep_add_lease(self):
4169         self.basedir = "web/Grid/deep_add_lease"
4170         self.set_up_grid(num_clients=2)
4171         c0 = self.g.clients[0]
4172         self.uris = {}
4173         self.fileurls = {}
4174         DATA = "data" * 100
4175         d = c0.create_dirnode()
4176         def _stash_root_and_create_file(n):
4177             self.rootnode = n
4178             self.uris["root"] = n.get_uri()
4179             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4180             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4181         d.addCallback(_stash_root_and_create_file)
4182         def _stash_uri(fn, which):
4183             self.uris[which] = fn.get_uri()
4184         d.addCallback(_stash_uri, "one")
4185         d.addCallback(lambda ign:
4186                       self.rootnode.add_file(u"small",
4187                                              upload.Data("literal",
4188                                                         convergence="")))
4189         d.addCallback(_stash_uri, "small")
4190
4191         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4192         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4193         d.addCallback(_stash_uri, "mutable")
4194
4195         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4196         def _done(res):
4197             units = [simplejson.loads(line)
4198                      for line in res.splitlines()
4199                      if line]
4200             # root, one, small, mutable,   stats
4201             self.failUnlessReallyEqual(len(units), 4+1)
4202         d.addCallback(_done)
4203
4204         d.addCallback(self._count_leases, "root")
4205         d.addCallback(self._assert_leasecount, 1)
4206         d.addCallback(self._count_leases, "one")
4207         d.addCallback(self._assert_leasecount, 1)
4208         d.addCallback(self._count_leases, "mutable")
4209         d.addCallback(self._assert_leasecount, 1)
4210
4211         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4212         d.addCallback(_done)
4213
4214         d.addCallback(self._count_leases, "root")
4215         d.addCallback(self._assert_leasecount, 1)
4216         d.addCallback(self._count_leases, "one")
4217         d.addCallback(self._assert_leasecount, 1)
4218         d.addCallback(self._count_leases, "mutable")
4219         d.addCallback(self._assert_leasecount, 1)
4220
4221         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4222                       clientnum=1)
4223         d.addCallback(_done)
4224
4225         d.addCallback(self._count_leases, "root")
4226         d.addCallback(self._assert_leasecount, 2)
4227         d.addCallback(self._count_leases, "one")
4228         d.addCallback(self._assert_leasecount, 2)
4229         d.addCallback(self._count_leases, "mutable")
4230         d.addCallback(self._assert_leasecount, 2)
4231
4232         d.addErrback(self.explain_web_error)
4233         return d
4234
4235
4236     def test_exceptions(self):
4237         self.basedir = "web/Grid/exceptions"
4238         self.set_up_grid(num_clients=1, num_servers=2)
4239         c0 = self.g.clients[0]
4240         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4241         self.fileurls = {}
4242         DATA = "data" * 100
4243         d = c0.create_dirnode()
4244         def _stash_root(n):
4245             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4246             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4247             return n
4248         d.addCallback(_stash_root)
4249         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4250         def _stash_bad(ur):
4251             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4252             self.delete_shares_numbered(ur.uri, range(1,10))
4253
4254             u = uri.from_string(ur.uri)
4255             u.key = testutil.flip_bit(u.key, 0)
4256             baduri = u.to_string()
4257             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4258         d.addCallback(_stash_bad)
4259         d.addCallback(lambda ign: c0.create_dirnode())
4260         def _mangle_dirnode_1share(n):
4261             u = n.get_uri()
4262             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4263             self.fileurls["dir-1share-json"] = url + "?t=json"
4264             self.delete_shares_numbered(u, range(1,10))
4265         d.addCallback(_mangle_dirnode_1share)
4266         d.addCallback(lambda ign: c0.create_dirnode())
4267         def _mangle_dirnode_0share(n):
4268             u = n.get_uri()
4269             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4270             self.fileurls["dir-0share-json"] = url + "?t=json"
4271             self.delete_shares_numbered(u, range(0,10))
4272         d.addCallback(_mangle_dirnode_0share)
4273
4274         # NotEnoughSharesError should be reported sensibly, with a
4275         # text/plain explanation of the problem, and perhaps some
4276         # information on which shares *could* be found.
4277
4278         d.addCallback(lambda ignored:
4279                       self.shouldHTTPError("GET unrecoverable",
4280                                            410, "Gone", "NoSharesError",
4281                                            self.GET, self.fileurls["0shares"]))
4282         def _check_zero_shares(body):
4283             self.failIf("<html>" in body, body)
4284             body = " ".join(body.strip().split())
4285             exp = ("NoSharesError: no shares could be found. "
4286                    "Zero shares usually indicates a corrupt URI, or that "
4287                    "no servers were connected, but it might also indicate "
4288                    "severe corruption. You should perform a filecheck on "
4289                    "this object to learn more. The full error message is: "
4290                    "no shares (need 3). Last failure: None")
4291             self.failUnlessReallyEqual(exp, body)
4292         d.addCallback(_check_zero_shares)
4293
4294
4295         d.addCallback(lambda ignored:
4296                       self.shouldHTTPError("GET 1share",
4297                                            410, "Gone", "NotEnoughSharesError",
4298                                            self.GET, self.fileurls["1share"]))
4299         def _check_one_share(body):
4300             self.failIf("<html>" in body, body)
4301             body = " ".join(body.strip().split())
4302             msgbase = ("NotEnoughSharesError: This indicates that some "
4303                        "servers were unavailable, or that shares have been "
4304                        "lost to server departure, hard drive failure, or disk "
4305                        "corruption. You should perform a filecheck on "
4306                        "this object to learn more. The full error message is:"
4307                        )
4308             msg1 = msgbase + (" ran out of shares:"
4309                               " complete=sh0"
4310                               " pending="
4311                               " overdue= unused= need 3. Last failure: None")
4312             msg2 = msgbase + (" ran out of shares:"
4313                               " complete="
4314                               " pending=Share(sh0-on-xgru5)"
4315                               " overdue= unused= need 3. Last failure: None")
4316             self.failUnless(body == msg1 or body == msg2, body)
4317         d.addCallback(_check_one_share)
4318
4319         d.addCallback(lambda ignored:
4320                       self.shouldHTTPError("GET imaginary",
4321                                            404, "Not Found", None,
4322                                            self.GET, self.fileurls["imaginary"]))
4323         def _missing_child(body):
4324             self.failUnless("No such child: imaginary" in body, body)
4325         d.addCallback(_missing_child)
4326
4327         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4328         def _check_0shares_dir_html(body):
4329             self.failUnless("<html>" in body, body)
4330             # we should see the regular page, but without the child table or
4331             # the dirops forms
4332             body = " ".join(body.strip().split())
4333             self.failUnlessIn('href="?t=info">More info on this directory',
4334                               body)
4335             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4336                    "could not be retrieved, because there were insufficient "
4337                    "good shares. This might indicate that no servers were "
4338                    "connected, insufficient servers were connected, the URI "
4339                    "was corrupt, or that shares have been lost due to server "
4340                    "departure, hard drive failure, or disk corruption. You "
4341                    "should perform a filecheck on this object to learn more.")
4342             self.failUnlessIn(exp, body)
4343             self.failUnlessIn("No upload forms: directory is unreadable", body)
4344         d.addCallback(_check_0shares_dir_html)
4345
4346         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4347         def _check_1shares_dir_html(body):
4348             # at some point, we'll split UnrecoverableFileError into 0-shares
4349             # and some-shares like we did for immutable files (since there
4350             # are different sorts of advice to offer in each case). For now,
4351             # they present the same way.
4352             self.failUnless("<html>" in body, body)
4353             body = " ".join(body.strip().split())
4354             self.failUnlessIn('href="?t=info">More info on this directory',
4355                               body)
4356             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4357                    "could not be retrieved, because there were insufficient "
4358                    "good shares. This might indicate that no servers were "
4359                    "connected, insufficient servers were connected, the URI "
4360                    "was corrupt, or that shares have been lost due to server "
4361                    "departure, hard drive failure, or disk corruption. You "
4362                    "should perform a filecheck on this object to learn more.")
4363             self.failUnlessIn(exp, body)
4364             self.failUnlessIn("No upload forms: directory is unreadable", body)
4365         d.addCallback(_check_1shares_dir_html)
4366
4367         d.addCallback(lambda ignored:
4368                       self.shouldHTTPError("GET dir-0share-json",
4369                                            410, "Gone", "UnrecoverableFileError",
4370                                            self.GET,
4371                                            self.fileurls["dir-0share-json"]))
4372         def _check_unrecoverable_file(body):
4373             self.failIf("<html>" in body, body)
4374             body = " ".join(body.strip().split())
4375             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4376                    "could not be retrieved, because there were insufficient "
4377                    "good shares. This might indicate that no servers were "
4378                    "connected, insufficient servers were connected, the URI "
4379                    "was corrupt, or that shares have been lost due to server "
4380                    "departure, hard drive failure, or disk corruption. You "
4381                    "should perform a filecheck on this object to learn more.")
4382             self.failUnlessReallyEqual(exp, body)
4383         d.addCallback(_check_unrecoverable_file)
4384
4385         d.addCallback(lambda ignored:
4386                       self.shouldHTTPError("GET dir-1share-json",
4387                                            410, "Gone", "UnrecoverableFileError",
4388                                            self.GET,
4389                                            self.fileurls["dir-1share-json"]))
4390         d.addCallback(_check_unrecoverable_file)
4391
4392         d.addCallback(lambda ignored:
4393                       self.shouldHTTPError("GET imaginary",
4394                                            404, "Not Found", None,
4395                                            self.GET, self.fileurls["imaginary"]))
4396
4397         # attach a webapi child that throws a random error, to test how it
4398         # gets rendered.
4399         w = c0.getServiceNamed("webish")
4400         w.root.putChild("ERRORBOOM", ErrorBoom())
4401
4402         # "Accept: */*" :        should get a text/html stack trace
4403         # "Accept: text/plain" : should get a text/plain stack trace
4404         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4405         # no Accept header:      should get a text/html stack trace
4406
4407         d.addCallback(lambda ignored:
4408                       self.shouldHTTPError("GET errorboom_html",
4409                                            500, "Internal Server Error", None,
4410                                            self.GET, "ERRORBOOM",
4411                                            headers={"accept": ["*/*"]}))
4412         def _internal_error_html1(body):
4413             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4414         d.addCallback(_internal_error_html1)
4415
4416         d.addCallback(lambda ignored:
4417                       self.shouldHTTPError("GET errorboom_text",
4418                                            500, "Internal Server Error", None,
4419                                            self.GET, "ERRORBOOM",
4420                                            headers={"accept": ["text/plain"]}))
4421         def _internal_error_text2(body):
4422             self.failIf("<html>" in body, body)
4423             self.failUnless(body.startswith("Traceback "), body)
4424         d.addCallback(_internal_error_text2)
4425
4426         CLI_accepts = "text/plain, application/octet-stream"
4427         d.addCallback(lambda ignored:
4428                       self.shouldHTTPError("GET errorboom_text",
4429                                            500, "Internal Server Error", None,
4430                                            self.GET, "ERRORBOOM",
4431                                            headers={"accept": [CLI_accepts]}))
4432         def _internal_error_text3(body):
4433             self.failIf("<html>" in body, body)
4434             self.failUnless(body.startswith("Traceback "), body)
4435         d.addCallback(_internal_error_text3)
4436
4437         d.addCallback(lambda ignored:
4438                       self.shouldHTTPError("GET errorboom_text",
4439                                            500, "Internal Server Error", None,
4440                                            self.GET, "ERRORBOOM"))
4441         def _internal_error_html4(body):
4442             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4443         d.addCallback(_internal_error_html4)
4444
4445         def _flush_errors(res):
4446             # Trial: please ignore the CompletelyUnhandledError in the logs
4447             self.flushLoggedErrors(CompletelyUnhandledError)
4448             return res
4449         d.addBoth(_flush_errors)
4450
4451         return d
4452
4453 class CompletelyUnhandledError(Exception):
4454     pass
4455 class ErrorBoom(rend.Page):
4456     def beforeRender(self, ctx):
4457         raise CompletelyUnhandledError("whoops")