src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17 import common_util as testutil
  18
  19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  20 # create a webserver that works against them
  21
  22 class FakeIntroducerClient:
  23     def get_all_connectors(self):
  24         return {}
  25     def get_all_connections_for(self, service_name):
  26         return frozenset()
  27     def get_all_peerids(self):
  28         return frozenset()
  29
  30 class FakeClient(service.MultiService):
  31     nodeid = "fake_nodeid"
  32     nickname = "fake_nickname"
  33     basedir = "fake_basedir"
  34     def get_versions(self):
  35         return {'allmydata': "fake",
  36                 'foolscap': "fake",
  37                 'twisted': "fake",
  38                 'zfec': "fake",
  39                 }
  40     introducer_furl = "None"
  41     introducer_client = FakeIntroducerClient()
  42     _all_upload_status = [upload.UploadStatus()]
  43     _all_download_status = [download.DownloadStatus()]
  44     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  45     _all_publish_statuses = [publish.PublishStatus()]
  46     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  47     convergence = "some random string"
  48
  49     def connected_to_introducer(self):
  50         return False
  51
  52     def get_nickname_for_peerid(self, peerid):
  53         return u"John Doe"
  54
  55     def get_permuted_peers(self, service_name, key):
  56         return []
  57
  58     def create_node_from_uri(self, auri):
  59         u = uri.from_string(auri)
  60         if (INewDirectoryURI.providedBy(u)
  61             or IReadonlyNewDirectoryURI.providedBy(u)):
  62             return FakeDirectoryNode(self).init_from_uri(u)
  63         if IFileURI.providedBy(u):
  64             return FakeCHKFileNode(u, self)
  65         assert IMutableFileURI.providedBy(u), u
  66         return FakeMutableFileNode(self).init_from_uri(u)
  67
  68     def create_empty_dirnode(self):
  69         n = FakeDirectoryNode(self)
  70         d = n.create()
  71         d.addCallback(lambda res: n)
  72         return d
  73
  74     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  75     def create_mutable_file(self, contents=""):
  76         n = FakeMutableFileNode(self)
  77         return n.create(contents)
  78
  79     def upload(self, uploadable):
  80         d = uploadable.get_size()
  81         d.addCallback(lambda size: uploadable.read(size))
  82         def _got_data(datav):
  83             data = "".join(datav)
  84             n = create_chk_filenode(self, data)
  85             results = upload.UploadResults()
  86             results.uri = n.get_uri()
  87             return results
  88         d.addCallback(_got_data)
  89         return d
  90
  91     def list_all_upload_statuses(self):
  92         return self._all_upload_status
  93     def list_all_download_statuses(self):
  94         return self._all_download_status
  95     def list_all_mapupdate_statuses(self):
  96         return self._all_mapupdate_statuses
  97     def list_all_publish_statuses(self):
  98         return self._all_publish_statuses
  99     def list_all_retrieve_statuses(self):
 100         return self._all_retrieve_statuses
 101     def list_all_helper_statuses(self):
 102         return []
 103
 104 class MyGetter(client.HTTPPageGetter):
 105     handleStatus_206 = lambda self: self.handleStatus_200()
 106
 107 class HTTPClientHEADFactory(client.HTTPClientFactory):
 108     protocol = MyGetter
 109
 110     def noPage(self, reason):
 111         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 112         # exception when the response to a HEAD request had no body (when in
 113         # fact they are defined to never have a body). This was fixed in
 114         # Twisted-8.0 . To work around this, we catch the
 115         # PartialDownloadError and make it disappear.
 116         if (reason.check(client.PartialDownloadError)
 117             and self.method.upper() == "HEAD"):
 118             self.page("")
 119             return
 120         return client.HTTPClientFactory.noPage(self, reason)
 121
 122 class HTTPClientGETFactory(client.HTTPClientFactory):
 123     protocol = MyGetter
 124
 125 class WebMixin(object):
 126     def setUp(self):
 127         self.s = FakeClient()
 128         self.s.startService()
 129         self.staticdir = self.mktemp()
 130         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 131         s.setServiceParent(self.s)
 132         self.webish_port = port = s.listener._port.getHost().port
 133         self.webish_url = "http://localhost:%d" % port
 134
 135         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 136         d = defer.DeferredList(l)
 137         def _then(res):
 138             self.public_root = res[0][1]
 139             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 140             self.public_url = "/uri/" + self.public_root.get_uri()
 141             self.private_root = res[1][1]
 142
 143             foo = res[2][1]
 144             self._foo_node = foo
 145             self._foo_uri = foo.get_uri()
 146             self._foo_readonly_uri = foo.get_readonly_uri()
 147             # NOTE: we ignore the deferred on all set_uri() calls, because we
 148             # know the fake nodes do these synchronously
 149             self.public_root.set_uri(u"foo", foo.get_uri())
 150
 151             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 152             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 153
 154             foo.set_uri(u"empty", res[3][1].get_uri())
 155             sub_uri = res[4][1].get_uri()
 156             self._sub_uri = sub_uri
 157             foo.set_uri(u"sub", sub_uri)
 158             sub = self.s.create_node_from_uri(sub_uri)
 159
 160             _ign, n, blocking_uri = self.makefile(1)
 161             foo.set_uri(u"blockingfile", blocking_uri)
 162
 163             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 164             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 165             # still think of it as an umlaut
 166             foo.set_uri(unicode_filename, self._bar_txt_uri)
 167
 168             _ign, n, baz_file = self.makefile(2)
 169             sub.set_uri(u"baz.txt", baz_file)
 170
 171             _ign, n, self._bad_file_uri = self.makefile(3)
 172             # this uri should not be downloadable
 173             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 174
 175             rodir = res[5][1]
 176             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 177             rodir.set_uri(u"nor", baz_file)
 178
 179             # public/
 180             # public/foo/
 181             # public/foo/bar.txt
 182             # public/foo/blockingfile
 183             # public/foo/empty/
 184             # public/foo/sub/
 185             # public/foo/sub/baz.txt
 186             # public/reedownlee/
 187             # public/reedownlee/nor
 188             self.NEWFILE_CONTENTS = "newfile contents\n"
 189
 190             return foo.get_metadata_for(u"bar.txt")
 191         d.addCallback(_then)
 192         def _got_metadata(metadata):
 193             self._bar_txt_metadata = metadata
 194         d.addCallback(_got_metadata)
 195         return d
 196
 197     def makefile(self, number):
 198         contents = "contents of file %s\n" % number
 199         n = create_chk_filenode(self.s, contents)
 200         return contents, n, n.get_uri()
 201
 202     def tearDown(self):
 203         return self.s.stopService()
 204
 205     def failUnlessIsBarDotTxt(self, res):
 206         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 207
 208     def failUnlessIsBarJSON(self, res):
 209         data = simplejson.loads(res)
 210         self.failUnless(isinstance(data, list))
 211         self.failUnlessEqual(data[0], u"filenode")
 212         self.failUnless(isinstance(data[1], dict))
 213         self.failIf(data[1]["mutable"])
 214         self.failIf("rw_uri" in data[1]) # immutable
 215         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 216         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 217
 218     def failUnlessIsFooJSON(self, res):
 219         data = simplejson.loads(res)
 220         self.failUnless(isinstance(data, list))
 221         self.failUnlessEqual(data[0], "dirnode", res)
 222         self.failUnless(isinstance(data[1], dict))
 223         self.failUnless(data[1]["mutable"])
 224         self.failUnless("rw_uri" in data[1]) # mutable
 225         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 226         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 227
 228         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 229         self.failUnlessEqual(kidnames,
 230                              [u"bar.txt", u"blockingfile", u"empty",
 231                               u"n\u00fc.txt", u"sub"])
 232         kids = dict( [(unicode(name),value)
 233                       for (name,value)
 234                       in data[1]["children"].iteritems()] )
 235         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 236         self.failUnless("metadata" in kids[u"sub"][1])
 237         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 238         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 239         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 242         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 243                              self._bar_txt_metadata["ctime"])
 244         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 245                              self._bar_txt_uri)
 246
 247     def GET(self, urlpath, followRedirect=False, return_response=False,
 248             **kwargs):
 249         # if return_response=True, this fires with (data, statuscode,
 250         # respheaders) instead of just data.
 251         assert not isinstance(urlpath, unicode)
 252         url = self.webish_url + urlpath
 253         factory = HTTPClientGETFactory(url, method="GET",
 254                                        followRedirect=followRedirect, **kwargs)
 255         reactor.connectTCP("localhost", self.webish_port, factory)
 256         d = factory.deferred
 257         def _got_data(data):
 258             return (data, factory.status, factory.response_headers)
 259         if return_response:
 260             d.addCallback(_got_data)
 261         return factory.deferred
 262
 263     def HEAD(self, urlpath, return_response=False, **kwargs):
 264         # this requires some surgery, because twisted.web.client doesn't want
 265         # to give us back the response headers.
 266         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 267         reactor.connectTCP("localhost", self.webish_port, factory)
 268         d = factory.deferred
 269         def _got_data(data):
 270             return (data, factory.status, factory.response_headers)
 271         if return_response:
 272             d.addCallback(_got_data)
 273         return factory.deferred
 274
 275     def PUT(self, urlpath, data, **kwargs):
 276         url = self.webish_url + urlpath
 277         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 278
 279     def DELETE(self, urlpath):
 280         url = self.webish_url + urlpath
 281         return client.getPage(url, method="DELETE")
 282
 283     def POST(self, urlpath, followRedirect=False, **fields):
 284         url = self.webish_url + urlpath
 285         sepbase = "boogabooga"
 286         sep = "--" + sepbase
 287         form = []
 288         form.append(sep)
 289         form.append('Content-Disposition: form-data; name="_charset"')
 290         form.append('')
 291         form.append('UTF-8')
 292         form.append(sep)
 293         for name, value in fields.iteritems():
 294             if isinstance(value, tuple):
 295                 filename, value = value
 296                 form.append('Content-Disposition: form-data; name="%s"; '
 297                             'filename="%s"' % (name, filename.encode("utf-8")))
 298             else:
 299                 form.append('Content-Disposition: form-data; name="%s"' % name)
 300             form.append('')
 301             if isinstance(value, unicode):
 302                 value = value.encode("utf-8")
 303             else:
 304                 value = str(value)
 305             assert isinstance(value, str)
 306             form.append(value)
 307             form.append(sep)
 308         form[-1] += "--"
 309         body = "\r\n".join(form) + "\r\n"
 310         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 311                    }
 312         return client.getPage(url, method="POST", postdata=body,
 313                               headers=headers, followRedirect=followRedirect)
 314
 315     def shouldFail(self, res, expected_failure, which,
 316                    substring=None, response_substring=None):
 317         if isinstance(res, failure.Failure):
 318             res.trap(expected_failure)
 319             if substring:
 320                 self.failUnless(substring in str(res),
 321                                 "substring '%s' not in '%s'"
 322                                 % (substring, str(res)))
 323             if response_substring:
 324                 self.failUnless(response_substring in res.value.response,
 325                                 "response substring '%s' not in '%s'"
 326                                 % (response_substring, res.value.response))
 327         else:
 328             self.fail("%s was supposed to raise %s, not get '%s'" %
 329                       (which, expected_failure, res))
 330
 331     def shouldFail2(self, expected_failure, which, substring,
 332                     response_substring,
 333                     callable, *args, **kwargs):
 334         assert substring is None or isinstance(substring, str)
 335         assert response_substring is None or isinstance(response_substring, str)
 336         d = defer.maybeDeferred(callable, *args, **kwargs)
 337         def done(res):
 338             if isinstance(res, failure.Failure):
 339                 res.trap(expected_failure)
 340                 if substring:
 341                     self.failUnless(substring in str(res),
 342                                     "%s: substring '%s' not in '%s'"
 343                                     % (which, substring, str(res)))
 344                 if response_substring:
 345                     self.failUnless(response_substring in res.value.response,
 346                                     "%s: response substring '%s' not in '%s'"
 347                                     % (which,
 348                                        response_substring, res.value.response))
 349             else:
 350                 self.fail("%s was supposed to raise %s, not get '%s'" %
 351                           (which, expected_failure, res))
 352         d.addBoth(done)
 353         return d
 354
 355     def should404(self, res, which):
 356         if isinstance(res, failure.Failure):
 357             res.trap(error.Error)
 358             self.failUnlessEqual(res.value.status, "404")
 359         else:
 360             self.fail("%s was supposed to Error(404), not get '%s'" %
 361                       (which, res))
 362
 363     def shouldHTTPError(self, res, which, code=None, substring=None,
 364                         response_substring=None):
 365         if isinstance(res, failure.Failure):
 366             res.trap(error.Error)
 367             if code is not None:
 368                 self.failUnlessEqual(res.value.status, str(code))
 369             if substring:
 370                 self.failUnless(substring in str(res),
 371                                 "substring '%s' not in '%s'"
 372                                 % (substring, str(res)))
 373             if response_substring:
 374                 self.failUnless(response_substring in res.value.response,
 375                                 "response substring '%s' not in '%s'"
 376                                 % (response_substring, res.value.response))
 377         else:
 378             self.fail("%s was supposed to Error(%s), not get '%s'" %
 379                       (which, code, res))
 380
 381     def shouldHTTPError2(self, which,
 382                          code=None, substring=None, response_substring=None,
 383                          callable=None, *args, **kwargs):
 384         assert substring is None or isinstance(substring, str)
 385         assert callable
 386         d = defer.maybeDeferred(callable, *args, **kwargs)
 387         d.addBoth(self.shouldHTTPError, which,
 388                   code, substring, response_substring)
 389         return d
 390
 391
 392 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 393     def test_create(self):
 394         pass
 395
 396     def test_welcome(self):
 397         d = self.GET("/")
 398         def _check(res):
 399             self.failUnless('Welcome To AllMyData' in res)
 400             self.failUnless('Tahoe' in res)
 401
 402             self.s.basedir = 'web/test_welcome'
 403             fileutil.make_dirs("web/test_welcome")
 404             fileutil.make_dirs("web/test_welcome/private")
 405             return self.GET("/")
 406         d.addCallback(_check)
 407         return d
 408
 409     def test_provisioning_math(self):
 410         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 411         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 412         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 413         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 414         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 415
 416     def test_provisioning(self):
 417         d = self.GET("/provisioning/")
 418         def _check(res):
 419             self.failUnless('Tahoe Provisioning Tool' in res)
 420             fields = {'filled': True,
 421                       "num_users": int(50e3),
 422                       "files_per_user": 1000,
 423                       "space_per_user": int(1e9),
 424                       "sharing_ratio": 1.0,
 425                       "encoding_parameters": "3-of-10-5",
 426                       "num_servers": 30,
 427                       "ownership_mode": "A",
 428                       "download_rate": 100,
 429                       "upload_rate": 10,
 430                       "delete_rate": 10,
 431                       "lease_timer": 7,
 432                       }
 433             return self.POST("/provisioning/", **fields)
 434
 435         d.addCallback(_check)
 436         def _check2(res):
 437             self.failUnless('Tahoe Provisioning Tool' in res)
 438             self.failUnless("Share space consumed: 167.01TB" in res)
 439
 440             fields = {'filled': True,
 441                       "num_users": int(50e6),
 442                       "files_per_user": 1000,
 443                       "space_per_user": int(5e9),
 444                       "sharing_ratio": 1.0,
 445                       "encoding_parameters": "25-of-100-50",
 446                       "num_servers": 30000,
 447                       "ownership_mode": "E",
 448                       "drive_failure_model": "U",
 449                       "drive_size": 1000,
 450                       "download_rate": 1000,
 451                       "upload_rate": 100,
 452                       "delete_rate": 100,
 453                       "lease_timer": 7,
 454                       }
 455             return self.POST("/provisioning/", **fields)
 456         d.addCallback(_check2)
 457         def _check3(res):
 458             self.failUnless("Share space consumed: huge!" in res)
 459             fields = {'filled': True}
 460             return self.POST("/provisioning/", **fields)
 461         d.addCallback(_check3)
 462         def _check4(res):
 463             self.failUnless("Share space consumed:" in res)
 464         d.addCallback(_check4)
 465         return d
 466
 467     def test_status(self):
 468         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 469         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 470         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 471         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 472         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 473         d = self.GET("/status", followRedirect=True)
 474         def _check(res):
 475             self.failUnless('Upload and Download Status' in res, res)
 476             self.failUnless('"down-%d"' % dl_num in res, res)
 477             self.failUnless('"up-%d"' % ul_num in res, res)
 478             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 479             self.failUnless('"publish-%d"' % pub_num in res, res)
 480             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 481         d.addCallback(_check)
 482         d.addCallback(lambda res: self.GET("/status/?t=json"))
 483         def _check_json(res):
 484             data = simplejson.loads(res)
 485             self.failUnless(isinstance(data, dict))
 486             active = data["active"]
 487             # TODO: test more. We need a way to fake an active operation
 488             # here.
 489         d.addCallback(_check_json)
 490
 491         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 492         def _check_dl(res):
 493             self.failUnless("File Download Status" in res, res)
 494         d.addCallback(_check_dl)
 495         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 496         def _check_ul(res):
 497             self.failUnless("File Upload Status" in res, res)
 498         d.addCallback(_check_ul)
 499         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 500         def _check_mapupdate(res):
 501             self.failUnless("Mutable File Servermap Update Status" in res, res)
 502         d.addCallback(_check_mapupdate)
 503         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 504         def _check_publish(res):
 505             self.failUnless("Mutable File Publish Status" in res, res)
 506         d.addCallback(_check_publish)
 507         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 508         def _check_retrieve(res):
 509             self.failUnless("Mutable File Retrieve Status" in res, res)
 510         d.addCallback(_check_retrieve)
 511
 512         return d
 513
 514     def test_status_numbers(self):
 515         drrm = status.DownloadResultsRendererMixin()
 516         self.failUnlessEqual(drrm.render_time(None, None), "")
 517         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 518         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 519         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 520         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 521         self.failUnlessEqual(drrm.render_rate(None, None), "")
 522         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 523         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 524         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 525
 526         urrm = status.UploadResultsRendererMixin()
 527         self.failUnlessEqual(urrm.render_time(None, None), "")
 528         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 529         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 530         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 531         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 532         self.failUnlessEqual(urrm.render_rate(None, None), "")
 533         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 534         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 535         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 536
 537     def test_GET_FILEURL(self):
 538         d = self.GET(self.public_url + "/foo/bar.txt")
 539         d.addCallback(self.failUnlessIsBarDotTxt)
 540         return d
 541
 542     def test_GET_FILEURL_range(self):
 543         headers = {"range": "bytes=1-10"}
 544         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 545                      return_response=True)
 546         def _got((res, status, headers)):
 547             self.failUnlessEqual(int(status), 206)
 548             self.failUnless(headers.has_key("content-range"))
 549             self.failUnlessEqual(headers["content-range"][0],
 550                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 551             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 552         d.addCallback(_got)
 553         return d
 554
 555     def test_GET_FILEURL_partial_range(self):
 556         headers = {"range": "bytes=5-"}
 557         length  = len(self.BAR_CONTENTS)
 558         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 559                      return_response=True)
 560         def _got((res, status, headers)):
 561             self.failUnlessEqual(int(status), 206)
 562             self.failUnless(headers.has_key("content-range"))
 563             self.failUnlessEqual(headers["content-range"][0],
 564                                  "bytes 5-%d/%d" % (length-1, length))
 565             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 566         d.addCallback(_got)
 567         return d
 568
 569     def test_HEAD_FILEURL_range(self):
 570         headers = {"range": "bytes=1-10"}
 571         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 572                      return_response=True)
 573         def _got((res, status, headers)):
 574             self.failUnlessEqual(res, "")
 575             self.failUnlessEqual(int(status), 206)
 576             self.failUnless(headers.has_key("content-range"))
 577             self.failUnlessEqual(headers["content-range"][0],
 578                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 579         d.addCallback(_got)
 580         return d
 581
 582     def test_HEAD_FILEURL_partial_range(self):
 583         headers = {"range": "bytes=5-"}
 584         length  = len(self.BAR_CONTENTS)
 585         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 586                      return_response=True)
 587         def _got((res, status, headers)):
 588             self.failUnlessEqual(int(status), 206)
 589             self.failUnless(headers.has_key("content-range"))
 590             self.failUnlessEqual(headers["content-range"][0],
 591                                  "bytes 5-%d/%d" % (length-1, length))
 592         d.addCallback(_got)
 593         return d
 594
 595     def test_GET_FILEURL_range_bad(self):
 596         headers = {"range": "BOGUS=fizbop-quarnak"}
 597         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 598                              "400 Bad Request",
 599                              "Syntactically invalid http range header",
 600                              self.GET, self.public_url + "/foo/bar.txt",
 601                              headers=headers)
 602         return d
 603
 604     def test_HEAD_FILEURL(self):
 605         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 606         def _got((res, status, headers)):
 607             self.failUnlessEqual(res, "")
 608             self.failUnlessEqual(headers["content-length"][0],
 609                                  str(len(self.BAR_CONTENTS)))
 610             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 611         d.addCallback(_got)
 612         return d
 613
 614     def test_GET_FILEURL_named(self):
 615         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 616         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 617         d = self.GET(base + "/@@name=/blah.txt")
 618         d.addCallback(self.failUnlessIsBarDotTxt)
 619         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 622         d.addCallback(self.failUnlessIsBarDotTxt)
 623         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 624         d.addCallback(self.failUnlessIsBarDotTxt)
 625         save_url = base + "?save=true&filename=blah.txt"
 626         d.addCallback(lambda res: self.GET(save_url))
 627         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 628         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 629         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 630         u_url = base + "?save=true&filename=" + u_fn_e
 631         d.addCallback(lambda res: self.GET(u_url))
 632         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 633         return d
 634
 635     def test_PUT_FILEURL_named_bad(self):
 636         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 637         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 638                              "400 Bad Request",
 639                              "/file can only be used with GET or HEAD",
 640                              self.PUT, base + "/@@name=/blah.txt", "")
 641         return d
 642
 643     def test_GET_DIRURL_named_bad(self):
 644         base = "/file/%s" % urllib.quote(self._foo_uri)
 645         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 646                              "400 Bad Request",
 647                              "is not a file-cap",
 648                              self.GET, base + "/@@name=/blah.txt")
 649         return d
 650
 651     def test_GET_slash_file_bad(self):
 652         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 653                              "404 Not Found",
 654                              "/file must be followed by a file-cap and a name",
 655                              self.GET, "/file")
 656         return d
 657
 658     def test_GET_unhandled_URI_named(self):
 659         contents, n, newuri = self.makefile(12)
 660         verifier_cap = n.get_verifier().to_string()
 661         base = "/file/%s" % urllib.quote(verifier_cap)
 662         # client.create_node_from_uri() can't handle verify-caps
 663         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 664                              "400 Bad Request",
 665                              "is not a valid file- or directory- cap",
 666                              self.GET, base)
 667         return d
 668
 669     def test_GET_unhandled_URI(self):
 670         contents, n, newuri = self.makefile(12)
 671         verifier_cap = n.get_verifier().to_string()
 672         base = "/uri/%s" % urllib.quote(verifier_cap)
 673         # client.create_node_from_uri() can't handle verify-caps
 674         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 675                              "400 Bad Request",
 676                              "is not a valid file- or directory- cap",
 677                              self.GET, base)
 678         return d
 679
 680     def test_GET_FILE_URI(self):
 681         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 682         d = self.GET(base)
 683         d.addCallback(self.failUnlessIsBarDotTxt)
 684         return d
 685
 686     def test_GET_FILE_URI_badchild(self):
 687         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 688         errmsg = "Files have no children, certainly not named 'boguschild'"
 689         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 690                              "400 Bad Request", errmsg,
 691                              self.GET, base)
 692         return d
 693
 694     def test_PUT_FILE_URI_badchild(self):
 695         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 696         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 697         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 698                              "400 Bad Request", errmsg,
 699                              self.PUT, base, "")
 700         return d
 701
 702     def test_GET_FILEURL_save(self):
 703         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 704         # TODO: look at the headers, expect a Content-Disposition: attachment
 705         # header.
 706         d.addCallback(self.failUnlessIsBarDotTxt)
 707         return d
 708
 709     def test_GET_FILEURL_missing(self):
 710         d = self.GET(self.public_url + "/foo/missing")
 711         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 712         return d
 713
 714     def test_PUT_NEWFILEURL(self):
 715         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 716         # TODO: we lose the response code, so we can't check this
 717         #self.failUnlessEqual(responsecode, 201)
 718         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 719         d.addCallback(lambda res:
 720                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 721                                                       self.NEWFILE_CONTENTS))
 722         return d
 723
 724     def test_PUT_NEWFILEURL_range_bad(self):
 725         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 726         target = self.public_url + "/foo/new.txt"
 727         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 728                              "501 Not Implemented",
 729                              "Content-Range in PUT not yet supported",
 730                              # (and certainly not for immutable files)
 731                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 732                              headers=headers)
 733         d.addCallback(lambda res:
 734                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 735         return d
 736
 737     def test_PUT_NEWFILEURL_mutable(self):
 738         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 739                      self.NEWFILE_CONTENTS)
 740         # TODO: we lose the response code, so we can't check this
 741         #self.failUnlessEqual(responsecode, 201)
 742         def _check_uri(res):
 743             u = uri.from_string_mutable_filenode(res)
 744             self.failUnless(u.is_mutable())
 745             self.failIf(u.is_readonly())
 746             return res
 747         d.addCallback(_check_uri)
 748         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 749         d.addCallback(lambda res:
 750                       self.failUnlessMutableChildContentsAre(self._foo_node,
 751                                                              u"new.txt",
 752                                                              self.NEWFILE_CONTENTS))
 753         return d
 754
 755     def test_PUT_NEWFILEURL_mutable_toobig(self):
 756         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 757                              "413 Request Entity Too Large",
 758                              "SDMF is limited to one segment, and 10001 > 10000",
 759                              self.PUT,
 760                              self.public_url + "/foo/new.txt?mutable=true",
 761                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 762         return d
 763
 764     def test_PUT_NEWFILEURL_replace(self):
 765         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 766         # TODO: we lose the response code, so we can't check this
 767         #self.failUnlessEqual(responsecode, 200)
 768         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 769         d.addCallback(lambda res:
 770                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 771                                                       self.NEWFILE_CONTENTS))
 772         return d
 773
 774     def test_PUT_NEWFILEURL_bad_t(self):
 775         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 776                              "PUT to a file: bad t=bogus",
 777                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 778                              "contents")
 779         return d
 780
 781     def test_PUT_NEWFILEURL_no_replace(self):
 782         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 783                      self.NEWFILE_CONTENTS)
 784         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 785                   "409 Conflict",
 786                   "There was already a child by that name, and you asked me "
 787                   "to not replace it")
 788         return d
 789
 790     def test_PUT_NEWFILEURL_mkdirs(self):
 791         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 792         fn = self._foo_node
 793         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 794         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 795         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 796         d.addCallback(lambda res:
 797                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 798                                                       self.NEWFILE_CONTENTS))
 799         return d
 800
 801     def test_PUT_NEWFILEURL_blocked(self):
 802         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 803                      self.NEWFILE_CONTENTS)
 804         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 805                   "409 Conflict",
 806                   "Unable to create directory 'blockingfile': a file was in the way")
 807         return d
 808
 809     def test_DELETE_FILEURL(self):
 810         d = self.DELETE(self.public_url + "/foo/bar.txt")
 811         d.addCallback(lambda res:
 812                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 813         return d
 814
 815     def test_DELETE_FILEURL_missing(self):
 816         d = self.DELETE(self.public_url + "/foo/missing")
 817         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 818         return d
 819
 820     def test_DELETE_FILEURL_missing2(self):
 821         d = self.DELETE(self.public_url + "/missing/missing")
 822         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 823         return d
 824
 825     def test_GET_FILEURL_json(self):
 826         # twisted.web.http.parse_qs ignores any query args without an '=', so
 827         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 828         # instead. This may make it tricky to emulate the S3 interface
 829         # completely.
 830         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 831         d.addCallback(self.failUnlessIsBarJSON)
 832         return d
 833
 834     def test_GET_FILEURL_json_missing(self):
 835         d = self.GET(self.public_url + "/foo/missing?json")
 836         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 837         return d
 838
 839     def test_GET_FILEURL_uri(self):
 840         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 841         def _check(res):
 842             self.failUnlessEqual(res, self._bar_txt_uri)
 843         d.addCallback(_check)
 844         d.addCallback(lambda res:
 845                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 846         def _check2(res):
 847             # for now, for files, uris and readonly-uris are the same
 848             self.failUnlessEqual(res, self._bar_txt_uri)
 849         d.addCallback(_check2)
 850         return d
 851
 852     def test_GET_FILEURL_badtype(self):
 853         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 854                                   "bad t=bogus",
 855                                   self.GET,
 856                                   self.public_url + "/foo/bar.txt?t=bogus")
 857         return d
 858
 859     def test_GET_FILEURL_uri_missing(self):
 860         d = self.GET(self.public_url + "/foo/missing?t=uri")
 861         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 862         return d
 863
 864     def test_GET_DIRURL(self):
 865         # the addSlash means we get a redirect here
 866         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 867         ROOT = "../../.."
 868         d = self.GET(self.public_url + "/foo", followRedirect=True)
 869         def _check(res):
 870             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 871                             in res, res)
 872             # the FILE reference points to a URI, but it should end in bar.txt
 873             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 874                        (ROOT, urllib.quote(self._bar_txt_uri)))
 875             get_bar = "".join([r'<td>',
 876                                r'<a href="%s">bar.txt</a>' % bar_url,
 877                                r'</td>',
 878                                r'\s+<td>FILE</td>',
 879                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 880                                ])
 881             self.failUnless(re.search(get_bar, res), res)
 882             for line in res.split("\n"):
 883                 # find the line that contains the delete button for bar.txt
 884                 if ("form action" in line and
 885                     'value="delete"' in line and
 886                     'value="bar.txt"' in line):
 887                     # the form target should use a relative URL
 888                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 889                     self.failUnless(('action="%s"' % foo_url) in line, line)
 890                     # and the when_done= should too
 891                     #done_url = urllib.quote(???)
 892                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 893                     #                in line, line)
 894                     break
 895             else:
 896                 self.fail("unable to find delete-bar.txt line", res)
 897
 898             # the DIR reference just points to a URI
 899             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 900             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 901                        + r'\s+<td>DIR</td>')
 902             self.failUnless(re.search(get_sub, res), res)
 903         d.addCallback(_check)
 904
 905         # look at a directory which is readonly
 906         d.addCallback(lambda res:
 907                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 908         def _check2(res):
 909             self.failUnless("(readonly)" in res, res)
 910             self.failIf("Upload a file" in res, res)
 911         d.addCallback(_check2)
 912
 913         # and at a directory that contains a readonly directory
 914         d.addCallback(lambda res:
 915                       self.GET(self.public_url, followRedirect=True))
 916         def _check3(res):
 917             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 918                                       '</td>\s+<td>DIR-RO</td>', res))
 919         d.addCallback(_check3)
 920
 921         return d
 922
 923     def test_GET_DIRURL_badtype(self):
 924         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 925                                   400, "Bad Request",
 926                                   "bad t=bogus",
 927                                   self.GET,
 928                                   self.public_url + "/foo?t=bogus")
 929         return d
 930
 931     def test_GET_DIRURL_json(self):
 932         d = self.GET(self.public_url + "/foo?t=json")
 933         d.addCallback(self.failUnlessIsFooJSON)
 934         return d
 935
 936
 937     def test_POST_DIRURL_manifest_no_ophandle(self):
 938         d = self.shouldFail2(error.Error,
 939                              "test_POST_DIRURL_manifest_no_ophandle",
 940                              "400 Bad Request",
 941                              "slow operation requires ophandle=",
 942                              self.POST, self.public_url, t="start-manifest")
 943         return d
 944
 945     def test_POST_DIRURL_manifest(self):
 946         d = defer.succeed(None)
 947         def getman(ignored, output):
 948             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 949                           followRedirect=True)
 950             d.addCallback(self.wait_for_operation, "125")
 951             d.addCallback(self.get_operation_results, "125", output)
 952             return d
 953         d.addCallback(getman, None)
 954         def _got_html(manifest):
 955             self.failUnless("Manifest of SI=" in manifest)
 956             self.failUnless("<td>sub</td>" in manifest)
 957             self.failUnless(self._sub_uri in manifest)
 958             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 959         d.addCallback(_got_html)
 960
 961         # both t=status and unadorned GET should be identical
 962         d.addCallback(lambda res: self.GET("/operations/125"))
 963         d.addCallback(_got_html)
 964
 965         d.addCallback(getman, "html")
 966         d.addCallback(_got_html)
 967         d.addCallback(getman, "text")
 968         def _got_text(manifest):
 969             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 970             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 971         d.addCallback(_got_text)
 972         d.addCallback(getman, "JSON")
 973         def _got_json(res):
 974             data = res["manifest"]
 975             got = {}
 976             for (path_list, cap) in data:
 977                 got[tuple(path_list)] = cap
 978             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 979             self.failUnless((u"sub",u"baz.txt") in got)
 980             self.failUnless("finished" in res)
 981             self.failUnless("origin" in res)
 982             self.failUnless("storage-index" in res)
 983             self.failUnless("verifycaps" in res)
 984             self.failUnless("stats" in res)
 985         d.addCallback(_got_json)
 986         return d
 987
 988     def test_POST_DIRURL_deepsize_no_ophandle(self):
 989         d = self.shouldFail2(error.Error,
 990                              "test_POST_DIRURL_deepsize_no_ophandle",
 991                              "400 Bad Request",
 992                              "slow operation requires ophandle=",
 993                              self.POST, self.public_url, t="start-deep-size")
 994         return d
 995
 996     def test_POST_DIRURL_deepsize(self):
 997         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 998                       followRedirect=True)
 999         d.addCallback(self.wait_for_operation, "126")
1000         d.addCallback(self.get_operation_results, "126", "json")
1001         def _got_json(data):
1002             self.failUnlessEqual(data["finished"], True)
1003             size = data["size"]
1004             self.failUnless(size > 1000)
1005         d.addCallback(_got_json)
1006         d.addCallback(self.get_operation_results, "126", "text")
1007         def _got_text(res):
1008             mo = re.search(r'^size: (\d+)$', res, re.M)
1009             self.failUnless(mo, res)
1010             size = int(mo.group(1))
1011             # with directories, the size varies.
1012             self.failUnless(size > 1000)
1013         d.addCallback(_got_text)
1014         return d
1015
1016     def test_POST_DIRURL_deepstats_no_ophandle(self):
1017         d = self.shouldFail2(error.Error,
1018                              "test_POST_DIRURL_deepstats_no_ophandle",
1019                              "400 Bad Request",
1020                              "slow operation requires ophandle=",
1021                              self.POST, self.public_url, t="start-deep-stats")
1022         return d
1023
1024     def test_POST_DIRURL_deepstats(self):
1025         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1026                       followRedirect=True)
1027         d.addCallback(self.wait_for_operation, "127")
1028         d.addCallback(self.get_operation_results, "127", "json")
1029         def _got_json(stats):
1030             expected = {"count-immutable-files": 3,
1031                         "count-mutable-files": 0,
1032                         "count-literal-files": 0,
1033                         "count-files": 3,
1034                         "count-directories": 3,
1035                         "size-immutable-files": 57,
1036                         "size-literal-files": 0,
1037                         #"size-directories": 1912, # varies
1038                         #"largest-directory": 1590,
1039                         "largest-directory-children": 5,
1040                         "largest-immutable-file": 19,
1041                         }
1042             for k,v in expected.iteritems():
1043                 self.failUnlessEqual(stats[k], v,
1044                                      "stats[%s] was %s, not %s" %
1045                                      (k, stats[k], v))
1046             self.failUnlessEqual(stats["size-files-histogram"],
1047                                  [ [11, 31, 3] ])
1048         d.addCallback(_got_json)
1049         return d
1050
1051     def test_GET_DIRURL_uri(self):
1052         d = self.GET(self.public_url + "/foo?t=uri")
1053         def _check(res):
1054             self.failUnlessEqual(res, self._foo_uri)
1055         d.addCallback(_check)
1056         return d
1057
1058     def test_GET_DIRURL_readonly_uri(self):
1059         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1060         def _check(res):
1061             self.failUnlessEqual(res, self._foo_readonly_uri)
1062         d.addCallback(_check)
1063         return d
1064
1065     def test_PUT_NEWDIRURL(self):
1066         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1067         d.addCallback(lambda res:
1068                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1069         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1070         d.addCallback(self.failUnlessNodeKeysAre, [])
1071         return d
1072
1073     def test_PUT_NEWDIRURL_exists(self):
1074         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1075         d.addCallback(lambda res:
1076                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1077         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1078         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1079         return d
1080
1081     def test_PUT_NEWDIRURL_blocked(self):
1082         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1083                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1084                              self.PUT,
1085                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1086         d.addCallback(lambda res:
1087                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1088         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1089         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1090         return d
1091
1092     def test_PUT_NEWDIRURL_mkdir_p(self):
1093         d = defer.succeed(None)
1094         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1095         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1096         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1097         def mkdir_p(mkpnode):
1098             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1099             d = self.POST(url)
1100             def made_subsub(ssuri):
1101                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1102                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1103                 d = self.POST(url)
1104                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1105                 return d
1106             d.addCallback(made_subsub)
1107             return d
1108         d.addCallback(mkdir_p)
1109         return d
1110
1111     def test_PUT_NEWDIRURL_mkdirs(self):
1112         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1113         d.addCallback(lambda res:
1114                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1115         d.addCallback(lambda res:
1116                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1117         d.addCallback(lambda res:
1118                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1119         d.addCallback(self.failUnlessNodeKeysAre, [])
1120         return d
1121
1122     def test_DELETE_DIRURL(self):
1123         d = self.DELETE(self.public_url + "/foo")
1124         d.addCallback(lambda res:
1125                       self.failIfNodeHasChild(self.public_root, u"foo"))
1126         return d
1127
1128     def test_DELETE_DIRURL_missing(self):
1129         d = self.DELETE(self.public_url + "/foo/missing")
1130         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1131         d.addCallback(lambda res:
1132                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1133         return d
1134
1135     def test_DELETE_DIRURL_missing2(self):
1136         d = self.DELETE(self.public_url + "/missing")
1137         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1138         return d
1139
1140     def dump_root(self):
1141         print "NODEWALK"
1142         w = webish.DirnodeWalkerMixin()
1143         def visitor(childpath, childnode, metadata):
1144             print childpath
1145         d = w.walk(self.public_root, visitor)
1146         return d
1147
1148     def failUnlessNodeKeysAre(self, node, expected_keys):
1149         for k in expected_keys:
1150             assert isinstance(k, unicode)
1151         d = node.list()
1152         def _check(children):
1153             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1154         d.addCallback(_check)
1155         return d
1156     def failUnlessNodeHasChild(self, node, name):
1157         assert isinstance(name, unicode)
1158         d = node.list()
1159         def _check(children):
1160             self.failUnless(name in children)
1161         d.addCallback(_check)
1162         return d
1163     def failIfNodeHasChild(self, node, name):
1164         assert isinstance(name, unicode)
1165         d = node.list()
1166         def _check(children):
1167             self.failIf(name in children)
1168         d.addCallback(_check)
1169         return d
1170
1171     def failUnlessChildContentsAre(self, node, name, expected_contents):
1172         assert isinstance(name, unicode)
1173         d = node.get_child_at_path(name)
1174         d.addCallback(lambda node: node.download_to_data())
1175         def _check(contents):
1176             self.failUnlessEqual(contents, expected_contents)
1177         d.addCallback(_check)
1178         return d
1179
1180     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1181         assert isinstance(name, unicode)
1182         d = node.get_child_at_path(name)
1183         d.addCallback(lambda node: node.download_best_version())
1184         def _check(contents):
1185             self.failUnlessEqual(contents, expected_contents)
1186         d.addCallback(_check)
1187         return d
1188
1189     def failUnlessChildURIIs(self, node, name, expected_uri):
1190         assert isinstance(name, unicode)
1191         d = node.get_child_at_path(name)
1192         def _check(child):
1193             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1194         d.addCallback(_check)
1195         return d
1196
1197     def failUnlessURIMatchesChild(self, got_uri, node, name):
1198         assert isinstance(name, unicode)
1199         d = node.get_child_at_path(name)
1200         def _check(child):
1201             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1202         d.addCallback(_check)
1203         return d
1204
1205     def failUnlessCHKURIHasContents(self, got_uri, contents):
1206         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1207
1208     def test_POST_upload(self):
1209         d = self.POST(self.public_url + "/foo", t="upload",
1210                       file=("new.txt", self.NEWFILE_CONTENTS))
1211         fn = self._foo_node
1212         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1213         d.addCallback(lambda res:
1214                       self.failUnlessChildContentsAre(fn, u"new.txt",
1215                                                       self.NEWFILE_CONTENTS))
1216         return d
1217
1218     def test_POST_upload_unicode(self):
1219         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1220         d = self.POST(self.public_url + "/foo", t="upload",
1221                       file=(filename, self.NEWFILE_CONTENTS))
1222         fn = self._foo_node
1223         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1224         d.addCallback(lambda res:
1225                       self.failUnlessChildContentsAre(fn, filename,
1226                                                       self.NEWFILE_CONTENTS))
1227         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1228         d.addCallback(lambda res: self.GET(target_url))
1229         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1230                                                             self.NEWFILE_CONTENTS,
1231                                                             contents))
1232         return d
1233
1234     def test_POST_upload_unicode_named(self):
1235         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1236         d = self.POST(self.public_url + "/foo", t="upload",
1237                       name=filename,
1238                       file=("overridden", self.NEWFILE_CONTENTS))
1239         fn = self._foo_node
1240         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1241         d.addCallback(lambda res:
1242                       self.failUnlessChildContentsAre(fn, filename,
1243                                                       self.NEWFILE_CONTENTS))
1244         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1245         d.addCallback(lambda res: self.GET(target_url))
1246         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1247                                                             self.NEWFILE_CONTENTS,
1248                                                             contents))
1249         return d
1250
1251     def test_POST_upload_no_link(self):
1252         d = self.POST("/uri", t="upload",
1253                       file=("new.txt", self.NEWFILE_CONTENTS))
1254         def _check_upload_results(page):
1255             # this should be a page which describes the results of the upload
1256             # that just finished.
1257             self.failUnless("Upload Results:" in page)
1258             self.failUnless("URI:" in page)
1259             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1260             mo = uri_re.search(page)
1261             self.failUnless(mo, page)
1262             new_uri = mo.group(1)
1263             return new_uri
1264         d.addCallback(_check_upload_results)
1265         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1266         return d
1267
1268     def test_POST_upload_no_link_whendone(self):
1269         d = self.POST("/uri", t="upload", when_done="/",
1270                       file=("new.txt", self.NEWFILE_CONTENTS))
1271         d.addBoth(self.shouldRedirect, "/")
1272         return d
1273
1274     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1275         d = defer.maybeDeferred(callable, *args, **kwargs)
1276         def done(res):
1277             if isinstance(res, failure.Failure):
1278                 res.trap(error.PageRedirect)
1279                 statuscode = res.value.status
1280                 target = res.value.location
1281                 return checker(statuscode, target)
1282             self.fail("%s: callable was supposed to redirect, not return '%s'"
1283                       % (which, res))
1284         d.addBoth(done)
1285         return d
1286
1287     def test_POST_upload_no_link_whendone_results(self):
1288         def check(statuscode, target):
1289             self.failUnlessEqual(statuscode, str(http.FOUND))
1290             self.failUnless(target.startswith(self.webish_url), target)
1291             return client.getPage(target, method="GET")
1292         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1293                                  check,
1294                                  self.POST, "/uri", t="upload",
1295                                  when_done="/uri/%(uri)s",
1296                                  file=("new.txt", self.NEWFILE_CONTENTS))
1297         d.addCallback(lambda res:
1298                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1299         return d
1300
1301     def test_POST_upload_no_link_mutable(self):
1302         d = self.POST("/uri", t="upload", mutable="true",
1303                       file=("new.txt", self.NEWFILE_CONTENTS))
1304         def _check(new_uri):
1305             new_uri = new_uri.strip()
1306             self.new_uri = new_uri
1307             u = IURI(new_uri)
1308             self.failUnless(IMutableFileURI.providedBy(u))
1309             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1310             n = self.s.create_node_from_uri(new_uri)
1311             return n.download_best_version()
1312         d.addCallback(_check)
1313         def _check2(data):
1314             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1315             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1316         d.addCallback(_check2)
1317         def _check3(data):
1318             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1319             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1320         d.addCallback(_check3)
1321         def _check4(data):
1322             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1323         d.addCallback(_check4)
1324         return d
1325
1326     def test_POST_upload_no_link_mutable_toobig(self):
1327         d = self.shouldFail2(error.Error,
1328                              "test_POST_upload_no_link_mutable_toobig",
1329                              "413 Request Entity Too Large",
1330                              "SDMF is limited to one segment, and 10001 > 10000",
1331                              self.POST,
1332                              "/uri", t="upload", mutable="true",
1333                              file=("new.txt",
1334                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1335         return d
1336
1337     def test_POST_upload_mutable(self):
1338         # this creates a mutable file
1339         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1340                       file=("new.txt", self.NEWFILE_CONTENTS))
1341         fn = self._foo_node
1342         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1343         d.addCallback(lambda res:
1344                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1345                                                              self.NEWFILE_CONTENTS))
1346         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1347         def _got(newnode):
1348             self.failUnless(IMutableFileNode.providedBy(newnode))
1349             self.failUnless(newnode.is_mutable())
1350             self.failIf(newnode.is_readonly())
1351             self._mutable_node = newnode
1352             self._mutable_uri = newnode.get_uri()
1353         d.addCallback(_got)
1354
1355         # now upload it again and make sure that the URI doesn't change
1356         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1357         d.addCallback(lambda res:
1358                       self.POST(self.public_url + "/foo", t="upload",
1359                                 mutable="true",
1360                                 file=("new.txt", NEWER_CONTENTS)))
1361         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1362         d.addCallback(lambda res:
1363                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1364                                                              NEWER_CONTENTS))
1365         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1366         def _got2(newnode):
1367             self.failUnless(IMutableFileNode.providedBy(newnode))
1368             self.failUnless(newnode.is_mutable())
1369             self.failIf(newnode.is_readonly())
1370             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1371         d.addCallback(_got2)
1372
1373         # upload a second time, using PUT instead of POST
1374         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1375         d.addCallback(lambda res:
1376                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1377         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1378         d.addCallback(lambda res:
1379                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1380                                                              NEW2_CONTENTS))
1381
1382         # finally list the directory, since mutable files are displayed
1383         # slightly differently
1384
1385         d.addCallback(lambda res:
1386                       self.GET(self.public_url + "/foo/",
1387                                followRedirect=True))
1388         def _check_page(res):
1389             # TODO: assert more about the contents
1390             self.failUnless("SSK" in res)
1391             return res
1392         d.addCallback(_check_page)
1393
1394         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1395         def _got3(newnode):
1396             self.failUnless(IMutableFileNode.providedBy(newnode))
1397             self.failUnless(newnode.is_mutable())
1398             self.failIf(newnode.is_readonly())
1399             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1400         d.addCallback(_got3)
1401
1402         # look at the JSON form of the enclosing directory
1403         d.addCallback(lambda res:
1404                       self.GET(self.public_url + "/foo/?t=json",
1405                                followRedirect=True))
1406         def _check_page_json(res):
1407             parsed = simplejson.loads(res)
1408             self.failUnlessEqual(parsed[0], "dirnode")
1409             children = dict( [(unicode(name),value)
1410                               for (name,value)
1411                               in parsed[1]["children"].iteritems()] )
1412             self.failUnless("new.txt" in children)
1413             new_json = children["new.txt"]
1414             self.failUnlessEqual(new_json[0], "filenode")
1415             self.failUnless(new_json[1]["mutable"])
1416             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1417             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1418             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1419         d.addCallback(_check_page_json)
1420
1421         # and the JSON form of the file
1422         d.addCallback(lambda res:
1423                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1424         def _check_file_json(res):
1425             parsed = simplejson.loads(res)
1426             self.failUnlessEqual(parsed[0], "filenode")
1427             self.failUnless(parsed[1]["mutable"])
1428             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1429             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1430             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1431         d.addCallback(_check_file_json)
1432
1433         # and look at t=uri and t=readonly-uri
1434         d.addCallback(lambda res:
1435                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1436         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1437         d.addCallback(lambda res:
1438                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1439         def _check_ro_uri(res):
1440             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1441             self.failUnlessEqual(res, ro_uri)
1442         d.addCallback(_check_ro_uri)
1443
1444         # make sure we can get to it from /uri/URI
1445         d.addCallback(lambda res:
1446                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1447         d.addCallback(lambda res:
1448                       self.failUnlessEqual(res, NEW2_CONTENTS))
1449
1450         # and that HEAD computes the size correctly
1451         d.addCallback(lambda res:
1452                       self.HEAD(self.public_url + "/foo/new.txt",
1453                                 return_response=True))
1454         def _got_headers((res, status, headers)):
1455             self.failUnlessEqual(res, "")
1456             self.failUnlessEqual(headers["content-length"][0],
1457                                  str(len(NEW2_CONTENTS)))
1458             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1459         d.addCallback(_got_headers)
1460
1461         # make sure that size errors are displayed correctly for overwrite
1462         d.addCallback(lambda res:
1463                       self.shouldFail2(error.Error,
1464                                        "test_POST_upload_mutable-toobig",
1465                                        "413 Request Entity Too Large",
1466                                        "SDMF is limited to one segment, and 10001 > 10000",
1467                                        self.POST,
1468                                        self.public_url + "/foo", t="upload",
1469                                        mutable="true",
1470                                        file=("new.txt",
1471                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1472                                        ))
1473
1474         d.addErrback(self.dump_error)
1475         return d
1476
1477     def test_POST_upload_mutable_toobig(self):
1478         d = self.shouldFail2(error.Error,
1479                              "test_POST_upload_no_link_mutable_toobig",
1480                              "413 Request Entity Too Large",
1481                              "SDMF is limited to one segment, and 10001 > 10000",
1482                              self.POST,
1483                              self.public_url + "/foo",
1484                              t="upload", mutable="true",
1485                              file=("new.txt",
1486                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1487         return d
1488
1489     def dump_error(self, f):
1490         # if the web server returns an error code (like 400 Bad Request),
1491         # web.client.getPage puts the HTTP response body into the .response
1492         # attribute of the exception object that it gives back. It does not
1493         # appear in the Failure's repr(), so the ERROR that trial displays
1494         # will be rather terse and unhelpful. addErrback this method to the
1495         # end of your chain to get more information out of these errors.
1496         if f.check(error.Error):
1497             print "web.error.Error:"
1498             print f
1499             print f.value.response
1500         return f
1501
1502     def test_POST_upload_replace(self):
1503         d = self.POST(self.public_url + "/foo", t="upload",
1504                       file=("bar.txt", self.NEWFILE_CONTENTS))
1505         fn = self._foo_node
1506         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1507         d.addCallback(lambda res:
1508                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1509                                                       self.NEWFILE_CONTENTS))
1510         return d
1511
1512     def test_POST_upload_no_replace_ok(self):
1513         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1514                       file=("new.txt", self.NEWFILE_CONTENTS))
1515         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1516         d.addCallback(lambda res: self.failUnlessEqual(res,
1517                                                        self.NEWFILE_CONTENTS))
1518         return d
1519
1520     def test_POST_upload_no_replace_queryarg(self):
1521         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1522                       file=("bar.txt", self.NEWFILE_CONTENTS))
1523         d.addBoth(self.shouldFail, error.Error,
1524                   "POST_upload_no_replace_queryarg",
1525                   "409 Conflict",
1526                   "There was already a child by that name, and you asked me "
1527                   "to not replace it")
1528         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1529         d.addCallback(self.failUnlessIsBarDotTxt)
1530         return d
1531
1532     def test_POST_upload_no_replace_field(self):
1533         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1534                       file=("bar.txt", self.NEWFILE_CONTENTS))
1535         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1536                   "409 Conflict",
1537                   "There was already a child by that name, and you asked me "
1538                   "to not replace it")
1539         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1540         d.addCallback(self.failUnlessIsBarDotTxt)
1541         return d
1542
1543     def test_POST_upload_whendone(self):
1544         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1545                       file=("new.txt", self.NEWFILE_CONTENTS))
1546         d.addBoth(self.shouldRedirect, "/THERE")
1547         fn = self._foo_node
1548         d.addCallback(lambda res:
1549                       self.failUnlessChildContentsAre(fn, u"new.txt",
1550                                                       self.NEWFILE_CONTENTS))
1551         return d
1552
1553     def test_POST_upload_named(self):
1554         fn = self._foo_node
1555         d = self.POST(self.public_url + "/foo", t="upload",
1556                       name="new.txt", file=self.NEWFILE_CONTENTS)
1557         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1558         d.addCallback(lambda res:
1559                       self.failUnlessChildContentsAre(fn, u"new.txt",
1560                                                       self.NEWFILE_CONTENTS))
1561         return d
1562
1563     def test_POST_upload_named_badfilename(self):
1564         d = self.POST(self.public_url + "/foo", t="upload",
1565                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1566         d.addBoth(self.shouldFail, error.Error,
1567                   "test_POST_upload_named_badfilename",
1568                   "400 Bad Request",
1569                   "name= may not contain a slash",
1570                   )
1571         # make sure that nothing was added
1572         d.addCallback(lambda res:
1573                       self.failUnlessNodeKeysAre(self._foo_node,
1574                                                  [u"bar.txt", u"blockingfile",
1575                                                   u"empty", u"n\u00fc.txt",
1576                                                   u"sub"]))
1577         return d
1578
1579     def test_POST_FILEURL_check(self):
1580         bar_url = self.public_url + "/foo/bar.txt"
1581         d = self.POST(bar_url, t="check")
1582         def _check(res):
1583             self.failUnless("Healthy :" in res)
1584         d.addCallback(_check)
1585         redir_url = "http://allmydata.org/TARGET"
1586         def _check2(statuscode, target):
1587             self.failUnlessEqual(statuscode, str(http.FOUND))
1588             self.failUnlessEqual(target, redir_url)
1589         d.addCallback(lambda res:
1590                       self.shouldRedirect2("test_POST_FILEURL_check",
1591                                            _check2,
1592                                            self.POST, bar_url,
1593                                            t="check",
1594                                            when_done=redir_url))
1595         d.addCallback(lambda res:
1596                       self.POST(bar_url, t="check", return_to=redir_url))
1597         def _check3(res):
1598             self.failUnless("Healthy :" in res)
1599             self.failUnless("Return to parent directory" in res)
1600             self.failUnless(redir_url in res)
1601         d.addCallback(_check3)
1602
1603         d.addCallback(lambda res:
1604                       self.POST(bar_url, t="check", output="JSON"))
1605         def _check_json(res):
1606             data = simplejson.loads(res)
1607             self.failUnless("storage-index" in data)
1608             self.failUnless(data["results"]["healthy"])
1609         d.addCallback(_check_json)
1610
1611         return d
1612
1613     def test_POST_FILEURL_check_and_repair(self):
1614         bar_url = self.public_url + "/foo/bar.txt"
1615         d = self.POST(bar_url, t="check", repair="true")
1616         def _check(res):
1617             self.failUnless("Healthy :" in res)
1618         d.addCallback(_check)
1619         redir_url = "http://allmydata.org/TARGET"
1620         def _check2(statuscode, target):
1621             self.failUnlessEqual(statuscode, str(http.FOUND))
1622             self.failUnlessEqual(target, redir_url)
1623         d.addCallback(lambda res:
1624                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1625                                            _check2,
1626                                            self.POST, bar_url,
1627                                            t="check", repair="true",
1628                                            when_done=redir_url))
1629         d.addCallback(lambda res:
1630                       self.POST(bar_url, t="check", return_to=redir_url))
1631         def _check3(res):
1632             self.failUnless("Healthy :" in res)
1633             self.failUnless("Return to parent directory" in res)
1634             self.failUnless(redir_url in res)
1635         d.addCallback(_check3)
1636         return d
1637
1638     def test_POST_DIRURL_check(self):
1639         foo_url = self.public_url + "/foo/"
1640         d = self.POST(foo_url, t="check")
1641         def _check(res):
1642             self.failUnless("Healthy :" in res, res)
1643         d.addCallback(_check)
1644         redir_url = "http://allmydata.org/TARGET"
1645         def _check2(statuscode, target):
1646             self.failUnlessEqual(statuscode, str(http.FOUND))
1647             self.failUnlessEqual(target, redir_url)
1648         d.addCallback(lambda res:
1649                       self.shouldRedirect2("test_POST_DIRURL_check",
1650                                            _check2,
1651                                            self.POST, foo_url,
1652                                            t="check",
1653                                            when_done=redir_url))
1654         d.addCallback(lambda res:
1655                       self.POST(foo_url, t="check", return_to=redir_url))
1656         def _check3(res):
1657             self.failUnless("Healthy :" in res, res)
1658             self.failUnless("Return to parent directory" in res)
1659             self.failUnless(redir_url in res)
1660         d.addCallback(_check3)
1661
1662         d.addCallback(lambda res:
1663                       self.POST(foo_url, t="check", output="JSON"))
1664         def _check_json(res):
1665             data = simplejson.loads(res)
1666             self.failUnless("storage-index" in data)
1667             self.failUnless(data["results"]["healthy"])
1668         d.addCallback(_check_json)
1669
1670         return d
1671
1672     def test_POST_DIRURL_check_and_repair(self):
1673         foo_url = self.public_url + "/foo/"
1674         d = self.POST(foo_url, t="check", repair="true")
1675         def _check(res):
1676             self.failUnless("Healthy :" in res, res)
1677         d.addCallback(_check)
1678         redir_url = "http://allmydata.org/TARGET"
1679         def _check2(statuscode, target):
1680             self.failUnlessEqual(statuscode, str(http.FOUND))
1681             self.failUnlessEqual(target, redir_url)
1682         d.addCallback(lambda res:
1683                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1684                                            _check2,
1685                                            self.POST, foo_url,
1686                                            t="check", repair="true",
1687                                            when_done=redir_url))
1688         d.addCallback(lambda res:
1689                       self.POST(foo_url, t="check", return_to=redir_url))
1690         def _check3(res):
1691             self.failUnless("Healthy :" in res)
1692             self.failUnless("Return to parent directory" in res)
1693             self.failUnless(redir_url in res)
1694         d.addCallback(_check3)
1695         return d
1696
1697     def wait_for_operation(self, ignored, ophandle):
1698         url = "/operations/" + ophandle
1699         url += "?t=status&output=JSON"
1700         d = self.GET(url)
1701         def _got(res):
1702             data = simplejson.loads(res)
1703             if not data["finished"]:
1704                 d = self.stall(delay=1.0)
1705                 d.addCallback(self.wait_for_operation, ophandle)
1706                 return d
1707             return data
1708         d.addCallback(_got)
1709         return d
1710
1711     def get_operation_results(self, ignored, ophandle, output=None):
1712         url = "/operations/" + ophandle
1713         url += "?t=status"
1714         if output:
1715             url += "&output=" + output
1716         d = self.GET(url)
1717         def _got(res):
1718             if output and output.lower() == "json":
1719                 return simplejson.loads(res)
1720             return res
1721         d.addCallback(_got)
1722         return d
1723
1724     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1725         d = self.shouldFail2(error.Error,
1726                              "test_POST_DIRURL_deepcheck_no_ophandle",
1727                              "400 Bad Request",
1728                              "slow operation requires ophandle=",
1729                              self.POST, self.public_url, t="start-deep-check")
1730         return d
1731
1732     def test_POST_DIRURL_deepcheck(self):
1733         def _check_redirect(statuscode, target):
1734             self.failUnlessEqual(statuscode, str(http.FOUND))
1735             self.failUnless(target.endswith("/operations/123"))
1736         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1737                                  self.POST, self.public_url,
1738                                  t="start-deep-check", ophandle="123")
1739         d.addCallback(self.wait_for_operation, "123")
1740         def _check_json(data):
1741             self.failUnlessEqual(data["finished"], True)
1742             self.failUnlessEqual(data["count-objects-checked"], 8)
1743             self.failUnlessEqual(data["count-objects-healthy"], 8)
1744         d.addCallback(_check_json)
1745         d.addCallback(self.get_operation_results, "123", "html")
1746         def _check_html(res):
1747             self.failUnless("Objects Checked: <span>8</span>" in res)
1748             self.failUnless("Objects Healthy: <span>8</span>" in res)
1749         d.addCallback(_check_html)
1750
1751         d.addCallback(lambda res:
1752                       self.GET("/operations/123/"))
1753         d.addCallback(_check_html) # should be the same as without the slash
1754
1755         d.addCallback(lambda res:
1756                       self.shouldFail2(error.Error, "one", "404 Not Found",
1757                                        "No detailed results for SI bogus",
1758                                        self.GET, "/operations/123/bogus"))
1759
1760         foo_si = self._foo_node.get_storage_index()
1761         foo_si_s = base32.b2a(foo_si)
1762         d.addCallback(lambda res:
1763                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1764         def _check_foo_json(res):
1765             data = simplejson.loads(res)
1766             self.failUnlessEqual(data["storage-index"], foo_si_s)
1767             self.failUnless(data["results"]["healthy"])
1768         d.addCallback(_check_foo_json)
1769         return d
1770
1771     def test_POST_DIRURL_deepcheck_and_repair(self):
1772         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1773                       ophandle="124", output="json", followRedirect=True)
1774         d.addCallback(self.wait_for_operation, "124")
1775         def _check_json(data):
1776             self.failUnlessEqual(data["finished"], True)
1777             self.failUnlessEqual(data["count-objects-checked"], 8)
1778             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1779             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1780             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1781             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1782             self.failUnlessEqual(data["count-repairs-successful"], 0)
1783             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1784             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1785             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1786             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1787         d.addCallback(_check_json)
1788         d.addCallback(self.get_operation_results, "124", "html")
1789         def _check_html(res):
1790             self.failUnless("Objects Checked: <span>8</span>" in res)
1791
1792             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1793             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1794             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1795
1796             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1797             self.failUnless("Repairs Successful: <span>0</span>" in res)
1798             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1799
1800             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1801             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1802             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1803         d.addCallback(_check_html)
1804         return d
1805
1806     def test_POST_FILEURL_bad_t(self):
1807         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1808                              "POST to file: bad t=bogus",
1809                              self.POST, self.public_url + "/foo/bar.txt",
1810                              t="bogus")
1811         return d
1812
1813     def test_POST_mkdir(self): # return value?
1814         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1815         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1816         d.addCallback(self.failUnlessNodeKeysAre, [])
1817         return d
1818
1819     def test_POST_mkdir_2(self):
1820         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1821         d.addCallback(lambda res:
1822                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1823         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1824         d.addCallback(self.failUnlessNodeKeysAre, [])
1825         return d
1826
1827     def test_POST_mkdirs_2(self):
1828         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1829         d.addCallback(lambda res:
1830                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1831         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1832         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1833         d.addCallback(self.failUnlessNodeKeysAre, [])
1834         return d
1835
1836     def test_POST_mkdir_no_parentdir_noredirect(self):
1837         d = self.POST("/uri?t=mkdir")
1838         def _after_mkdir(res):
1839             uri.NewDirectoryURI.init_from_string(res)
1840         d.addCallback(_after_mkdir)
1841         return d
1842
1843     def test_POST_mkdir_no_parentdir_redirect(self):
1844         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1845         d.addBoth(self.shouldRedirect, None, statuscode='303')
1846         def _check_target(target):
1847             target = urllib.unquote(target)
1848             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1849         d.addCallback(_check_target)
1850         return d
1851
1852     def test_POST_noparent_bad(self):
1853         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1854                                   "/uri accepts only PUT, PUT?t=mkdir, "
1855                                   "POST?t=upload, and POST?t=mkdir",
1856                                   self.POST, "/uri?t=bogus")
1857         return d
1858
1859     def test_welcome_page_mkdir_button(self):
1860         # Fetch the welcome page.
1861         d = self.GET("/")
1862         def _after_get_welcome_page(res):
1863             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1864             mo = MKDIR_BUTTON_RE.search(res)
1865             formaction = mo.group(1)
1866             formt = mo.group(2)
1867             formaname = mo.group(3)
1868             formavalue = mo.group(4)
1869             return (formaction, formt, formaname, formavalue)
1870         d.addCallback(_after_get_welcome_page)
1871         def _after_parse_form(res):
1872             (formaction, formt, formaname, formavalue) = res
1873             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1874         d.addCallback(_after_parse_form)
1875         d.addBoth(self.shouldRedirect, None, statuscode='303')
1876         return d
1877
1878     def test_POST_mkdir_replace(self): # return value?
1879         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1880         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1881         d.addCallback(self.failUnlessNodeKeysAre, [])
1882         return d
1883
1884     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1885         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1886         d.addBoth(self.shouldFail, error.Error,
1887                   "POST_mkdir_no_replace_queryarg",
1888                   "409 Conflict",
1889                   "There was already a child by that name, and you asked me "
1890                   "to not replace it")
1891         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1892         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1893         return d
1894
1895     def test_POST_mkdir_no_replace_field(self): # return value?
1896         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1897                       replace="false")
1898         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1899                   "409 Conflict",
1900                   "There was already a child by that name, and you asked me "
1901                   "to not replace it")
1902         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1903         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1904         return d
1905
1906     def test_POST_mkdir_whendone_field(self):
1907         d = self.POST(self.public_url + "/foo",
1908                       t="mkdir", name="newdir", when_done="/THERE")
1909         d.addBoth(self.shouldRedirect, "/THERE")
1910         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1911         d.addCallback(self.failUnlessNodeKeysAre, [])
1912         return d
1913
1914     def test_POST_mkdir_whendone_queryarg(self):
1915         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1916                       t="mkdir", name="newdir")
1917         d.addBoth(self.shouldRedirect, "/THERE")
1918         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1919         d.addCallback(self.failUnlessNodeKeysAre, [])
1920         return d
1921
1922     def test_POST_bad_t(self):
1923         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1924                              "POST to a directory with bad t=BOGUS",
1925                              self.POST, self.public_url + "/foo", t="BOGUS")
1926         return d
1927
1928     def test_POST_set_children(self):
1929         contents9, n9, newuri9 = self.makefile(9)
1930         contents10, n10, newuri10 = self.makefile(10)
1931         contents11, n11, newuri11 = self.makefile(11)
1932
1933         reqbody = """{
1934                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1935                                                 "size": 0,
1936                                                 "metadata": {
1937                                                   "ctime": 1002777696.7564139,
1938                                                   "mtime": 1002777696.7564139
1939                                                  }
1940                                                } ],
1941                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1942                                                 "size": 1,
1943                                                 "metadata": {
1944                                                   "ctime": 1002777696.7564139,
1945                                                   "mtime": 1002777696.7564139
1946                                                  }
1947                                                } ],
1948                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1949                                                 "size": 2,
1950                                                 "metadata": {
1951                                                   "ctime": 1002777696.7564139,
1952                                                   "mtime": 1002777696.7564139
1953                                                  }
1954                                                } ]
1955                     }""" % (newuri9, newuri10, newuri11)
1956
1957         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1958
1959         d = client.getPage(url, method="POST", postdata=reqbody)
1960         def _then(res):
1961             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1962             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1963             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1964
1965         d.addCallback(_then)
1966         d.addErrback(self.dump_error)
1967         return d
1968
1969     def test_POST_put_uri(self):
1970         contents, n, newuri = self.makefile(8)
1971         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1972         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1973         d.addCallback(lambda res:
1974                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1975                                                       contents))
1976         return d
1977
1978     def test_POST_put_uri_replace(self):
1979         contents, n, newuri = self.makefile(8)
1980         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1981         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1982         d.addCallback(lambda res:
1983                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1984                                                       contents))
1985         return d
1986
1987     def test_POST_put_uri_no_replace_queryarg(self):
1988         contents, n, newuri = self.makefile(8)
1989         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1990                       name="bar.txt", uri=newuri)
1991         d.addBoth(self.shouldFail, error.Error,
1992                   "POST_put_uri_no_replace_queryarg",
1993                   "409 Conflict",
1994                   "There was already a child by that name, and you asked me "
1995                   "to not replace it")
1996         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1997         d.addCallback(self.failUnlessIsBarDotTxt)
1998         return d
1999
2000     def test_POST_put_uri_no_replace_field(self):
2001         contents, n, newuri = self.makefile(8)
2002         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2003                       name="bar.txt", uri=newuri)
2004         d.addBoth(self.shouldFail, error.Error,
2005                   "POST_put_uri_no_replace_field",
2006                   "409 Conflict",
2007                   "There was already a child by that name, and you asked me "
2008                   "to not replace it")
2009         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2010         d.addCallback(self.failUnlessIsBarDotTxt)
2011         return d
2012
2013     def test_POST_delete(self):
2014         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2015         d.addCallback(lambda res: self._foo_node.list())
2016         def _check(children):
2017             self.failIf(u"bar.txt" in children)
2018         d.addCallback(_check)
2019         return d
2020
2021     def test_POST_rename_file(self):
2022         d = self.POST(self.public_url + "/foo", t="rename",
2023                       from_name="bar.txt", to_name='wibble.txt')
2024         d.addCallback(lambda res:
2025                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2026         d.addCallback(lambda res:
2027                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2028         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2029         d.addCallback(self.failUnlessIsBarDotTxt)
2030         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2031         d.addCallback(self.failUnlessIsBarJSON)
2032         return d
2033
2034     def test_POST_rename_file_redundant(self):
2035         d = self.POST(self.public_url + "/foo", t="rename",
2036                       from_name="bar.txt", to_name='bar.txt')
2037         d.addCallback(lambda res:
2038                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2039         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2040         d.addCallback(self.failUnlessIsBarDotTxt)
2041         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2042         d.addCallback(self.failUnlessIsBarJSON)
2043         return d
2044
2045     def test_POST_rename_file_replace(self):
2046         # rename a file and replace a directory with it
2047         d = self.POST(self.public_url + "/foo", t="rename",
2048                       from_name="bar.txt", to_name='empty')
2049         d.addCallback(lambda res:
2050                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2051         d.addCallback(lambda res:
2052                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2053         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2054         d.addCallback(self.failUnlessIsBarDotTxt)
2055         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2056         d.addCallback(self.failUnlessIsBarJSON)
2057         return d
2058
2059     def test_POST_rename_file_no_replace_queryarg(self):
2060         # rename a file and replace a directory with it
2061         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2062                       from_name="bar.txt", to_name='empty')
2063         d.addBoth(self.shouldFail, error.Error,
2064                   "POST_rename_file_no_replace_queryarg",
2065                   "409 Conflict",
2066                   "There was already a child by that name, and you asked me "
2067                   "to not replace it")
2068         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2069         d.addCallback(self.failUnlessIsEmptyJSON)
2070         return d
2071
2072     def test_POST_rename_file_no_replace_field(self):
2073         # rename a file and replace a directory with it
2074         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2075                       from_name="bar.txt", to_name='empty')
2076         d.addBoth(self.shouldFail, error.Error,
2077                   "POST_rename_file_no_replace_field",
2078                   "409 Conflict",
2079                   "There was already a child by that name, and you asked me "
2080                   "to not replace it")
2081         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2082         d.addCallback(self.failUnlessIsEmptyJSON)
2083         return d
2084
2085     def failUnlessIsEmptyJSON(self, res):
2086         data = simplejson.loads(res)
2087         self.failUnlessEqual(data[0], "dirnode", data)
2088         self.failUnlessEqual(len(data[1]["children"]), 0)
2089
2090     def test_POST_rename_file_slash_fail(self):
2091         d = self.POST(self.public_url + "/foo", t="rename",
2092                       from_name="bar.txt", to_name='kirk/spock.txt')
2093         d.addBoth(self.shouldFail, error.Error,
2094                   "test_POST_rename_file_slash_fail",
2095                   "400 Bad Request",
2096                   "to_name= may not contain a slash",
2097                   )
2098         d.addCallback(lambda res:
2099                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2100         return d
2101
2102     def test_POST_rename_dir(self):
2103         d = self.POST(self.public_url, t="rename",
2104                       from_name="foo", to_name='plunk')
2105         d.addCallback(lambda res:
2106                       self.failIfNodeHasChild(self.public_root, u"foo"))
2107         d.addCallback(lambda res:
2108                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2109         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2110         d.addCallback(self.failUnlessIsFooJSON)
2111         return d
2112
2113     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2114         """ If target is not None then the redirection has to go to target.  If
2115         statuscode is not None then the redirection has to be accomplished with
2116         that HTTP status code."""
2117         if not isinstance(res, failure.Failure):
2118             to_where = (target is None) and "somewhere" or ("to " + target)
2119             self.fail("%s: we were expecting to get redirected %s, not get an"
2120                       " actual page: %s" % (which, to_where, res))
2121         res.trap(error.PageRedirect)
2122         if statuscode is not None:
2123             self.failUnlessEqual(res.value.status, statuscode,
2124                                  "%s: not a redirect" % which)
2125         if target is not None:
2126             # the PageRedirect does not seem to capture the uri= query arg
2127             # properly, so we can't check for it.
2128             realtarget = self.webish_url + target
2129             self.failUnlessEqual(res.value.location, realtarget,
2130                                  "%s: wrong target" % which)
2131         return res.value.location
2132
2133     def test_GET_URI_form(self):
2134         base = "/uri?uri=%s" % self._bar_txt_uri
2135         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2136         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2137         d = self.GET(base)
2138         d.addBoth(self.shouldRedirect, targetbase)
2139         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2140         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2141         d.addCallback(lambda res: self.GET(base+"&t=json"))
2142         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2143         d.addCallback(self.log, "about to get file by uri")
2144         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2145         d.addCallback(self.failUnlessIsBarDotTxt)
2146         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2147         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2148                                            followRedirect=True))
2149         d.addCallback(self.failUnlessIsFooJSON)
2150         d.addCallback(self.log, "got dir by uri")
2151
2152         return d
2153
2154     def test_GET_URI_form_bad(self):
2155         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2156                              "400 Bad Request", "GET /uri requires uri=",
2157                              self.GET, "/uri")
2158         return d
2159
2160     def test_GET_rename_form(self):
2161         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2162                      followRedirect=True)
2163         def _check(res):
2164             self.failUnless('name="when_done" value="."' in res, res)
2165             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2166         d.addCallback(_check)
2167         return d
2168
2169     def log(self, res, msg):
2170         #print "MSG: %s  RES: %s" % (msg, res)
2171         log.msg(msg)
2172         return res
2173
2174     def test_GET_URI_URL(self):
2175         base = "/uri/%s" % self._bar_txt_uri
2176         d = self.GET(base)
2177         d.addCallback(self.failUnlessIsBarDotTxt)
2178         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2179         d.addCallback(self.failUnlessIsBarDotTxt)
2180         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2181         d.addCallback(self.failUnlessIsBarDotTxt)
2182         return d
2183
2184     def test_GET_URI_URL_dir(self):
2185         base = "/uri/%s?t=json" % self._foo_uri
2186         d = self.GET(base)
2187         d.addCallback(self.failUnlessIsFooJSON)
2188         return d
2189
2190     def test_GET_URI_URL_missing(self):
2191         base = "/uri/%s" % self._bad_file_uri
2192         d = self.GET(base)
2193         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2194                   http.GONE, response_substring="NotEnoughSharesError")
2195         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2196         # here? we must arrange for a download to fail after target.open()
2197         # has been called, and then inspect the response to see that it is
2198         # shorter than we expected.
2199         return d
2200
2201     def test_PUT_DIRURL_uri(self):
2202         d = self.s.create_empty_dirnode()
2203         def _made_dir(dn):
2204             new_uri = dn.get_uri()
2205             # replace /foo with a new (empty) directory
2206             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2207             d.addCallback(lambda res:
2208                           self.failUnlessEqual(res.strip(), new_uri))
2209             d.addCallback(lambda res:
2210                           self.failUnlessChildURIIs(self.public_root,
2211                                                     u"foo",
2212                                                     new_uri))
2213             return d
2214         d.addCallback(_made_dir)
2215         return d
2216
2217     def test_PUT_DIRURL_uri_noreplace(self):
2218         d = self.s.create_empty_dirnode()
2219         def _made_dir(dn):
2220             new_uri = dn.get_uri()
2221             # replace /foo with a new (empty) directory, but ask that
2222             # replace=false, so it should fail
2223             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2224                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2225                                  self.PUT,
2226                                  self.public_url + "/foo?t=uri&replace=false",
2227                                  new_uri)
2228             d.addCallback(lambda res:
2229                           self.failUnlessChildURIIs(self.public_root,
2230                                                     u"foo",
2231                                                     self._foo_uri))
2232             return d
2233         d.addCallback(_made_dir)
2234         return d
2235
2236     def test_PUT_DIRURL_bad_t(self):
2237         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2238                                  "400 Bad Request", "PUT to a directory",
2239                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2240         d.addCallback(lambda res:
2241                       self.failUnlessChildURIIs(self.public_root,
2242                                                 u"foo",
2243                                                 self._foo_uri))
2244         return d
2245
2246     def test_PUT_NEWFILEURL_uri(self):
2247         contents, n, new_uri = self.makefile(8)
2248         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2249         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2250         d.addCallback(lambda res:
2251                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2252                                                       contents))
2253         return d
2254
2255     def test_PUT_NEWFILEURL_uri_replace(self):
2256         contents, n, new_uri = self.makefile(8)
2257         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2258         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2259         d.addCallback(lambda res:
2260                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2261                                                       contents))
2262         return d
2263
2264     def test_PUT_NEWFILEURL_uri_no_replace(self):
2265         contents, n, new_uri = self.makefile(8)
2266         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2267         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2268                   "409 Conflict",
2269                   "There was already a child by that name, and you asked me "
2270                   "to not replace it")
2271         return d
2272
2273     def test_PUT_NEWFILE_URI(self):
2274         file_contents = "New file contents here\n"
2275         d = self.PUT("/uri", file_contents)
2276         def _check(uri):
2277             self.failUnless(uri in FakeCHKFileNode.all_contents)
2278             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2279                                  file_contents)
2280             return self.GET("/uri/%s" % uri)
2281         d.addCallback(_check)
2282         def _check2(res):
2283             self.failUnlessEqual(res, file_contents)
2284         d.addCallback(_check2)
2285         return d
2286
2287     def test_PUT_NEWFILE_URI_only_PUT(self):
2288         d = self.PUT("/uri?t=bogus", "")
2289         d.addBoth(self.shouldFail, error.Error,
2290                   "PUT_NEWFILE_URI_only_PUT",
2291                   "400 Bad Request",
2292                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2293         return d
2294
2295     def test_PUT_NEWFILE_URI_mutable(self):
2296         file_contents = "New file contents here\n"
2297         d = self.PUT("/uri?mutable=true", file_contents)
2298         def _check_mutable(uri):
2299             uri = uri.strip()
2300             u = IURI(uri)
2301             self.failUnless(IMutableFileURI.providedBy(u))
2302             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2303             n = self.s.create_node_from_uri(uri)
2304             return n.download_best_version()
2305         d.addCallback(_check_mutable)
2306         def _check2_mutable(data):
2307             self.failUnlessEqual(data, file_contents)
2308         d.addCallback(_check2_mutable)
2309         return d
2310
2311         def _check(uri):
2312             self.failUnless(uri in FakeCHKFileNode.all_contents)
2313             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2314                                  file_contents)
2315             return self.GET("/uri/%s" % uri)
2316         d.addCallback(_check)
2317         def _check2(res):
2318             self.failUnlessEqual(res, file_contents)
2319         d.addCallback(_check2)
2320         return d
2321
2322     def test_PUT_mkdir(self):
2323         d = self.PUT("/uri?t=mkdir", "")
2324         def _check(uri):
2325             n = self.s.create_node_from_uri(uri.strip())
2326             d2 = self.failUnlessNodeKeysAre(n, [])
2327             d2.addCallback(lambda res:
2328                            self.GET("/uri/%s?t=json" % uri))
2329             return d2
2330         d.addCallback(_check)
2331         d.addCallback(self.failUnlessIsEmptyJSON)
2332         return d
2333
2334     def test_POST_check(self):
2335         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2336         def _done(res):
2337             # this returns a string form of the results, which are probably
2338             # None since we're using fake filenodes.
2339             # TODO: verify that the check actually happened, by changing
2340             # FakeCHKFileNode to count how many times .check() has been
2341             # called.
2342             pass
2343         d.addCallback(_done)
2344         return d
2345
2346     def test_bad_method(self):
2347         url = self.webish_url + self.public_url + "/foo/bar.txt"
2348         d = self.shouldHTTPError2("test_bad_method",
2349                                   501, "Not Implemented",
2350                                   "I don't know how to treat a BOGUS request.",
2351                                   client.getPage, url, method="BOGUS")
2352         return d
2353
2354     def test_short_url(self):
2355         url = self.webish_url + "/uri"
2356         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2357                                   "I don't know how to treat a DELETE request.",
2358                                   client.getPage, url, method="DELETE")
2359         return d
2360
2361     def test_ophandle_bad(self):
2362         url = self.webish_url + "/operations/bogus?t=status"
2363         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2364                                   "unknown/expired handle 'bogus'",
2365                                   client.getPage, url)
2366         return d
2367
2368     def test_ophandle_cancel(self):
2369         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2370                       followRedirect=True)
2371         d.addCallback(lambda ignored:
2372                       self.GET("/operations/128?t=status&output=JSON"))
2373         def _check1(res):
2374             data = simplejson.loads(res)
2375             self.failUnless("finished" in data, res)
2376             monitor = self.ws.root.child_operations.handles["128"][0]
2377             d = self.POST("/operations/128?t=cancel&output=JSON")
2378             def _check2(res):
2379                 data = simplejson.loads(res)
2380                 self.failUnless("finished" in data, res)
2381                 # t=cancel causes the handle to be forgotten
2382                 self.failUnless(monitor.is_cancelled())
2383             d.addCallback(_check2)
2384             return d
2385         d.addCallback(_check1)
2386         d.addCallback(lambda ignored:
2387                       self.shouldHTTPError2("test_ophandle_cancel",
2388                                             404, "404 Not Found",
2389                                             "unknown/expired handle '128'",
2390                                             self.GET,
2391                                             "/operations/128?t=status&output=JSON"))
2392         return d
2393
2394     def test_ophandle_retainfor(self):
2395         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2396                       followRedirect=True)
2397         d.addCallback(lambda ignored:
2398                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2399         def _check1(res):
2400             data = simplejson.loads(res)
2401             self.failUnless("finished" in data, res)
2402         d.addCallback(_check1)
2403         # the retain-for=0 will cause the handle to be expired very soon
2404         d.addCallback(self.stall, 2.0)
2405         d.addCallback(lambda ignored:
2406                       self.shouldHTTPError2("test_ophandle_retainfor",
2407                                             404, "404 Not Found",
2408                                             "unknown/expired handle '129'",
2409                                             self.GET,
2410                                             "/operations/129?t=status&output=JSON"))
2411         return d
2412
2413     def test_ophandle_release_after_complete(self):
2414         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2415                       followRedirect=True)
2416         d.addCallback(self.wait_for_operation, "130")
2417         d.addCallback(lambda ignored:
2418                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2419         # the release-after-complete=true will cause the handle to be expired
2420         d.addCallback(lambda ignored:
2421                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2422                                             404, "404 Not Found",
2423                                             "unknown/expired handle '130'",
2424                                             self.GET,
2425                                             "/operations/130?t=status&output=JSON"))
2426         return d
2427
2428     def test_incident(self):
2429         d = self.POST("/report_incident", details="eek")
2430         def _done(res):
2431             self.failUnless("Thank you for your report!" in res, res)
2432         d.addCallback(_done)
2433         return d
2434
2435     def test_static(self):
2436         webdir = os.path.join(self.staticdir, "subdir")
2437         fileutil.make_dirs(webdir)
2438         f = open(os.path.join(webdir, "hello.txt"), "wb")
2439         f.write("hello")
2440         f.close()
2441
2442         d = self.GET("/static/subdir/hello.txt")
2443         def _check(res):
2444             self.failUnlessEqual(res, "hello")
2445         d.addCallback(_check)
2446         return d
2447
2448
2449 class Util(unittest.TestCase):
2450     def test_abbreviate_time(self):
2451         self.failUnlessEqual(common.abbreviate_time(None), "")
2452         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2453         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2454         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2455         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2456
2457     def test_abbreviate_rate(self):
2458         self.failUnlessEqual(common.abbreviate_rate(None), "")
2459         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2460         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2461         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2462
2463     def test_abbreviate_size(self):
2464         self.failUnlessEqual(common.abbreviate_size(None), "")
2465         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2466         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2467         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2468         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2469