src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, testutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17
  18 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  19 # create a webserver that works against them
  20
  21 class FakeIntroducerClient:
  22     def get_all_connectors(self):
  23         return {}
  24     def get_all_connections_for(self, service_name):
  25         return frozenset()
  26     def get_all_peerids(self):
  27         return frozenset()
  28
  29 class FakeClient(service.MultiService):
  30     nodeid = "fake_nodeid"
  31     nickname = "fake_nickname"
  32     basedir = "fake_basedir"
  33     def get_versions(self):
  34         return {'allmydata': "fake",
  35                 'foolscap': "fake",
  36                 'twisted': "fake",
  37                 'zfec': "fake",
  38                 }
  39     introducer_furl = "None"
  40     introducer_client = FakeIntroducerClient()
  41     _all_upload_status = [upload.UploadStatus()]
  42     _all_download_status = [download.DownloadStatus()]
  43     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  44     _all_publish_statuses = [publish.PublishStatus()]
  45     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  46     convergence = "some random string"
  47
  48     def connected_to_introducer(self):
  49         return False
  50
  51     def get_nickname_for_peerid(self, peerid):
  52         return u"John Doe"
  53
  54     def create_node_from_uri(self, auri):
  55         u = uri.from_string(auri)
  56         if (INewDirectoryURI.providedBy(u)
  57             or IReadonlyNewDirectoryURI.providedBy(u)):
  58             return FakeDirectoryNode(self).init_from_uri(u)
  59         if IFileURI.providedBy(u):
  60             return FakeCHKFileNode(u, self)
  61         assert IMutableFileURI.providedBy(u), u
  62         return FakeMutableFileNode(self).init_from_uri(u)
  63
  64     def create_empty_dirnode(self):
  65         n = FakeDirectoryNode(self)
  66         d = n.create()
  67         d.addCallback(lambda res: n)
  68         return d
  69
  70     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  71     def create_mutable_file(self, contents=""):
  72         n = FakeMutableFileNode(self)
  73         return n.create(contents)
  74
  75     def upload(self, uploadable):
  76         d = uploadable.get_size()
  77         d.addCallback(lambda size: uploadable.read(size))
  78         def _got_data(datav):
  79             data = "".join(datav)
  80             n = create_chk_filenode(self, data)
  81             results = upload.UploadResults()
  82             results.uri = n.get_uri()
  83             return results
  84         d.addCallback(_got_data)
  85         return d
  86
  87     def list_all_upload_statuses(self):
  88         return self._all_upload_status
  89     def list_all_download_statuses(self):
  90         return self._all_download_status
  91     def list_all_mapupdate_statuses(self):
  92         return self._all_mapupdate_statuses
  93     def list_all_publish_statuses(self):
  94         return self._all_publish_statuses
  95     def list_all_retrieve_statuses(self):
  96         return self._all_retrieve_statuses
  97     def list_all_helper_statuses(self):
  98         return []
  99
 100 class MyGetter(client.HTTPPageGetter):
 101     handleStatus_206 = lambda self: self.handleStatus_200()
 102
 103 class HTTPClientHEADFactory(client.HTTPClientFactory):
 104     protocol = MyGetter
 105
 106     def noPage(self, reason):
 107         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 108         # exception when the response to a HEAD request had no body (when in
 109         # fact they are defined to never have a body). This was fixed in
 110         # Twisted-8.0 . To work around this, we catch the
 111         # PartialDownloadError and make it disappear.
 112         if (reason.check(client.PartialDownloadError)
 113             and self.method.upper() == "HEAD"):
 114             self.page("")
 115             return
 116         return client.HTTPClientFactory.noPage(self, reason)
 117
 118 class HTTPClientGETFactory(client.HTTPClientFactory):
 119     protocol = MyGetter
 120
 121 class WebMixin(object):
 122     def setUp(self):
 123         self.s = FakeClient()
 124         self.s.startService()
 125         self.ws = s = webish.WebishServer("0")
 126         s.setServiceParent(self.s)
 127         self.webish_port = port = s.listener._port.getHost().port
 128         self.webish_url = "http://localhost:%d" % port
 129
 130         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 131         d = defer.DeferredList(l)
 132         def _then(res):
 133             self.public_root = res[0][1]
 134             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 135             self.public_url = "/uri/" + self.public_root.get_uri()
 136             self.private_root = res[1][1]
 137
 138             foo = res[2][1]
 139             self._foo_node = foo
 140             self._foo_uri = foo.get_uri()
 141             self._foo_readonly_uri = foo.get_readonly_uri()
 142             # NOTE: we ignore the deferred on all set_uri() calls, because we
 143             # know the fake nodes do these synchronously
 144             self.public_root.set_uri(u"foo", foo.get_uri())
 145
 146             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 147             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 148
 149             foo.set_uri(u"empty", res[3][1].get_uri())
 150             sub_uri = res[4][1].get_uri()
 151             self._sub_uri = sub_uri
 152             foo.set_uri(u"sub", sub_uri)
 153             sub = self.s.create_node_from_uri(sub_uri)
 154
 155             _ign, n, blocking_uri = self.makefile(1)
 156             foo.set_uri(u"blockingfile", blocking_uri)
 157
 158             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 159             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 160             # still think of it as an umlaut
 161             foo.set_uri(unicode_filename, self._bar_txt_uri)
 162
 163             _ign, n, baz_file = self.makefile(2)
 164             sub.set_uri(u"baz.txt", baz_file)
 165
 166             _ign, n, self._bad_file_uri = self.makefile(3)
 167             # this uri should not be downloadable
 168             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 169
 170             rodir = res[5][1]
 171             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 172             rodir.set_uri(u"nor", baz_file)
 173
 174             # public/
 175             # public/foo/
 176             # public/foo/bar.txt
 177             # public/foo/blockingfile
 178             # public/foo/empty/
 179             # public/foo/sub/
 180             # public/foo/sub/baz.txt
 181             # public/reedownlee/
 182             # public/reedownlee/nor
 183             self.NEWFILE_CONTENTS = "newfile contents\n"
 184
 185             return foo.get_metadata_for(u"bar.txt")
 186         d.addCallback(_then)
 187         def _got_metadata(metadata):
 188             self._bar_txt_metadata = metadata
 189         d.addCallback(_got_metadata)
 190         return d
 191
 192     def makefile(self, number):
 193         contents = "contents of file %s\n" % number
 194         n = create_chk_filenode(self.s, contents)
 195         return contents, n, n.get_uri()
 196
 197     def tearDown(self):
 198         return self.s.stopService()
 199
 200     def failUnlessIsBarDotTxt(self, res):
 201         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 202
 203     def failUnlessIsBarJSON(self, res):
 204         data = simplejson.loads(res)
 205         self.failUnless(isinstance(data, list))
 206         self.failUnlessEqual(data[0], u"filenode")
 207         self.failUnless(isinstance(data[1], dict))
 208         self.failIf(data[1]["mutable"])
 209         self.failIf("rw_uri" in data[1]) # immutable
 210         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 211         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 212
 213     def failUnlessIsFooJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], "dirnode", res)
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failUnless(data[1]["mutable"])
 219         self.failUnless("rw_uri" in data[1]) # mutable
 220         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 221         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 222
 223         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 224         self.failUnlessEqual(kidnames,
 225                              [u"bar.txt", u"blockingfile", u"empty",
 226                               u"n\u00fc.txt", u"sub"])
 227         kids = dict( [(unicode(name),value)
 228                       for (name,value)
 229                       in data[1]["children"].iteritems()] )
 230         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 231         self.failUnless("metadata" in kids[u"sub"][1])
 232         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 233         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 234         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 235         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 236         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 237         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 238                              self._bar_txt_metadata["ctime"])
 239         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 240                              self._bar_txt_uri)
 241
 242     def GET(self, urlpath, followRedirect=False, return_response=False,
 243             **kwargs):
 244         # if return_response=True, this fires with (data, statuscode,
 245         # respheaders) instead of just data.
 246         assert not isinstance(urlpath, unicode)
 247         url = self.webish_url + urlpath
 248         factory = HTTPClientGETFactory(url, method="GET",
 249                                        followRedirect=followRedirect, **kwargs)
 250         reactor.connectTCP("localhost", self.webish_port, factory)
 251         d = factory.deferred
 252         def _got_data(data):
 253             return (data, factory.status, factory.response_headers)
 254         if return_response:
 255             d.addCallback(_got_data)
 256         return factory.deferred
 257
 258     def HEAD(self, urlpath, return_response=False, **kwargs):
 259         # this requires some surgery, because twisted.web.client doesn't want
 260         # to give us back the response headers.
 261         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 262         reactor.connectTCP("localhost", self.webish_port, factory)
 263         d = factory.deferred
 264         def _got_data(data):
 265             return (data, factory.status, factory.response_headers)
 266         if return_response:
 267             d.addCallback(_got_data)
 268         return factory.deferred
 269
 270     def PUT(self, urlpath, data, **kwargs):
 271         url = self.webish_url + urlpath
 272         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 273
 274     def DELETE(self, urlpath):
 275         url = self.webish_url + urlpath
 276         return client.getPage(url, method="DELETE")
 277
 278     def POST(self, urlpath, followRedirect=False, **fields):
 279         url = self.webish_url + urlpath
 280         sepbase = "boogabooga"
 281         sep = "--" + sepbase
 282         form = []
 283         form.append(sep)
 284         form.append('Content-Disposition: form-data; name="_charset"')
 285         form.append('')
 286         form.append('UTF-8')
 287         form.append(sep)
 288         for name, value in fields.iteritems():
 289             if isinstance(value, tuple):
 290                 filename, value = value
 291                 form.append('Content-Disposition: form-data; name="%s"; '
 292                             'filename="%s"' % (name, filename.encode("utf-8")))
 293             else:
 294                 form.append('Content-Disposition: form-data; name="%s"' % name)
 295             form.append('')
 296             if isinstance(value, unicode):
 297                 value = value.encode("utf-8")
 298             else:
 299                 value = str(value)
 300             assert isinstance(value, str)
 301             form.append(value)
 302             form.append(sep)
 303         form[-1] += "--"
 304         body = "\r\n".join(form) + "\r\n"
 305         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 306                    }
 307         return client.getPage(url, method="POST", postdata=body,
 308                               headers=headers, followRedirect=followRedirect)
 309
 310     def shouldFail(self, res, expected_failure, which,
 311                    substring=None, response_substring=None):
 312         if isinstance(res, failure.Failure):
 313             res.trap(expected_failure)
 314             if substring:
 315                 self.failUnless(substring in str(res),
 316                                 "substring '%s' not in '%s'"
 317                                 % (substring, str(res)))
 318             if response_substring:
 319                 self.failUnless(response_substring in res.value.response,
 320                                 "response substring '%s' not in '%s'"
 321                                 % (response_substring, res.value.response))
 322         else:
 323             self.fail("%s was supposed to raise %s, not get '%s'" %
 324                       (which, expected_failure, res))
 325
 326     def shouldFail2(self, expected_failure, which, substring,
 327                     response_substring,
 328                     callable, *args, **kwargs):
 329         assert substring is None or isinstance(substring, str)
 330         assert response_substring is None or isinstance(response_substring, str)
 331         d = defer.maybeDeferred(callable, *args, **kwargs)
 332         def done(res):
 333             if isinstance(res, failure.Failure):
 334                 res.trap(expected_failure)
 335                 if substring:
 336                     self.failUnless(substring in str(res),
 337                                     "%s: substring '%s' not in '%s'"
 338                                     % (which, substring, str(res)))
 339                 if response_substring:
 340                     self.failUnless(response_substring in res.value.response,
 341                                     "%s: response substring '%s' not in '%s'"
 342                                     % (which,
 343                                        response_substring, res.value.response))
 344             else:
 345                 self.fail("%s was supposed to raise %s, not get '%s'" %
 346                           (which, expected_failure, res))
 347         d.addBoth(done)
 348         return d
 349
 350     def should404(self, res, which):
 351         if isinstance(res, failure.Failure):
 352             res.trap(error.Error)
 353             self.failUnlessEqual(res.value.status, "404")
 354         else:
 355             self.fail("%s was supposed to Error(404), not get '%s'" %
 356                       (which, res))
 357
 358     def shouldHTTPError(self, res, which, code=None, substring=None,
 359                         response_substring=None):
 360         if isinstance(res, failure.Failure):
 361             res.trap(error.Error)
 362             if code is not None:
 363                 self.failUnlessEqual(res.value.status, str(code))
 364             if substring:
 365                 self.failUnless(substring in str(res),
 366                                 "substring '%s' not in '%s'"
 367                                 % (substring, str(res)))
 368             if response_substring:
 369                 self.failUnless(response_substring in res.value.response,
 370                                 "response substring '%s' not in '%s'"
 371                                 % (response_substring, res.value.response))
 372         else:
 373             self.fail("%s was supposed to Error(%s), not get '%s'" %
 374                       (which, code, res))
 375
 376     def shouldHTTPError2(self, which,
 377                          code=None, substring=None, response_substring=None,
 378                          callable=None, *args, **kwargs):
 379         assert substring is None or isinstance(substring, str)
 380         assert callable
 381         d = defer.maybeDeferred(callable, *args, **kwargs)
 382         d.addBoth(self.shouldHTTPError, which,
 383                   code, substring, response_substring)
 384         return d
 385
 386
 387 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 388     def test_create(self):
 389         pass
 390
 391     def test_welcome(self):
 392         d = self.GET("/")
 393         def _check(res):
 394             self.failUnless('Welcome To AllMyData' in res)
 395             self.failUnless('Tahoe' in res)
 396
 397             self.s.basedir = 'web/test_welcome'
 398             fileutil.make_dirs("web/test_welcome")
 399             fileutil.make_dirs("web/test_welcome/private")
 400             return self.GET("/")
 401         d.addCallback(_check)
 402         return d
 403
 404     def test_provisioning_math(self):
 405         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 406         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 407         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 408         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 409         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 410
 411     def test_provisioning(self):
 412         d = self.GET("/provisioning/")
 413         def _check(res):
 414             self.failUnless('Tahoe Provisioning Tool' in res)
 415             fields = {'filled': True,
 416                       "num_users": int(50e3),
 417                       "files_per_user": 1000,
 418                       "space_per_user": int(1e9),
 419                       "sharing_ratio": 1.0,
 420                       "encoding_parameters": "3-of-10-5",
 421                       "num_servers": 30,
 422                       "ownership_mode": "A",
 423                       "download_rate": 100,
 424                       "upload_rate": 10,
 425                       "delete_rate": 10,
 426                       "lease_timer": 7,
 427                       }
 428             return self.POST("/provisioning/", **fields)
 429
 430         d.addCallback(_check)
 431         def _check2(res):
 432             self.failUnless('Tahoe Provisioning Tool' in res)
 433             self.failUnless("Share space consumed: 167.01TB" in res)
 434
 435             fields = {'filled': True,
 436                       "num_users": int(50e6),
 437                       "files_per_user": 1000,
 438                       "space_per_user": int(5e9),
 439                       "sharing_ratio": 1.0,
 440                       "encoding_parameters": "25-of-100-50",
 441                       "num_servers": 30000,
 442                       "ownership_mode": "E",
 443                       "drive_failure_model": "U",
 444                       "drive_size": 1000,
 445                       "download_rate": 1000,
 446                       "upload_rate": 100,
 447                       "delete_rate": 100,
 448                       "lease_timer": 7,
 449                       }
 450             return self.POST("/provisioning/", **fields)
 451         d.addCallback(_check2)
 452         def _check3(res):
 453             self.failUnless("Share space consumed: huge!" in res)
 454             fields = {'filled': True}
 455             return self.POST("/provisioning/", **fields)
 456         d.addCallback(_check3)
 457         def _check4(res):
 458             self.failUnless("Share space consumed:" in res)
 459         d.addCallback(_check4)
 460         return d
 461
 462     def test_status(self):
 463         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 464         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 465         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 466         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 467         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 468         d = self.GET("/status", followRedirect=True)
 469         def _check(res):
 470             self.failUnless('Upload and Download Status' in res, res)
 471             self.failUnless('"down-%d"' % dl_num in res, res)
 472             self.failUnless('"up-%d"' % ul_num in res, res)
 473             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 474             self.failUnless('"publish-%d"' % pub_num in res, res)
 475             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 476         d.addCallback(_check)
 477         d.addCallback(lambda res: self.GET("/status/?t=json"))
 478         def _check_json(res):
 479             data = simplejson.loads(res)
 480             self.failUnless(isinstance(data, dict))
 481             active = data["active"]
 482             # TODO: test more. We need a way to fake an active operation
 483             # here.
 484         d.addCallback(_check_json)
 485
 486         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 487         def _check_dl(res):
 488             self.failUnless("File Download Status" in res, res)
 489         d.addCallback(_check_dl)
 490         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 491         def _check_ul(res):
 492             self.failUnless("File Upload Status" in res, res)
 493         d.addCallback(_check_ul)
 494         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 495         def _check_mapupdate(res):
 496             self.failUnless("Mutable File Servermap Update Status" in res, res)
 497         d.addCallback(_check_mapupdate)
 498         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 499         def _check_publish(res):
 500             self.failUnless("Mutable File Publish Status" in res, res)
 501         d.addCallback(_check_publish)
 502         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 503         def _check_retrieve(res):
 504             self.failUnless("Mutable File Retrieve Status" in res, res)
 505         d.addCallback(_check_retrieve)
 506
 507         return d
 508
 509     def test_status_numbers(self):
 510         drrm = status.DownloadResultsRendererMixin()
 511         self.failUnlessEqual(drrm.render_time(None, None), "")
 512         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 513         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 514         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 515         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 516         self.failUnlessEqual(drrm.render_rate(None, None), "")
 517         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 518         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 519         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 520
 521         urrm = status.UploadResultsRendererMixin()
 522         self.failUnlessEqual(urrm.render_time(None, None), "")
 523         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 524         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 525         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 526         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 527         self.failUnlessEqual(urrm.render_rate(None, None), "")
 528         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 529         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 530         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 531
 532     def test_GET_FILEURL(self):
 533         d = self.GET(self.public_url + "/foo/bar.txt")
 534         d.addCallback(self.failUnlessIsBarDotTxt)
 535         return d
 536
 537     def test_GET_FILEURL_range(self):
 538         headers = {"range": "bytes=1-10"}
 539         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 540                      return_response=True)
 541         def _got((res, status, headers)):
 542             self.failUnlessEqual(int(status), 206)
 543             self.failUnless(headers.has_key("content-range"))
 544             self.failUnlessEqual(headers["content-range"][0],
 545                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 546             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 547         d.addCallback(_got)
 548         return d
 549
 550     def test_HEAD_FILEURL_range(self):
 551         headers = {"range": "bytes=1-10"}
 552         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 553                      return_response=True)
 554         def _got((res, status, headers)):
 555             self.failUnlessEqual(res, "")
 556             self.failUnlessEqual(int(status), 206)
 557             self.failUnless(headers.has_key("content-range"))
 558             self.failUnlessEqual(headers["content-range"][0],
 559                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 560         d.addCallback(_got)
 561         return d
 562
 563     def test_GET_FILEURL_range_bad(self):
 564         headers = {"range": "BOGUS=fizbop-quarnak"}
 565         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 566                              "400 Bad Request",
 567                              "Syntactically invalid http range header",
 568                              self.GET, self.public_url + "/foo/bar.txt",
 569                              headers=headers)
 570         return d
 571
 572     def test_HEAD_FILEURL(self):
 573         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 574         def _got((res, status, headers)):
 575             self.failUnlessEqual(res, "")
 576             self.failUnlessEqual(headers["content-length"][0],
 577                                  str(len(self.BAR_CONTENTS)))
 578             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 579         d.addCallback(_got)
 580         return d
 581
 582     def test_GET_FILEURL_named(self):
 583         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 584         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 585         d = self.GET(base + "/@@name=/blah.txt")
 586         d.addCallback(self.failUnlessIsBarDotTxt)
 587         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 588         d.addCallback(self.failUnlessIsBarDotTxt)
 589         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 590         d.addCallback(self.failUnlessIsBarDotTxt)
 591         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 592         d.addCallback(self.failUnlessIsBarDotTxt)
 593         save_url = base + "?save=true&filename=blah.txt"
 594         d.addCallback(lambda res: self.GET(save_url))
 595         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 596         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 597         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 598         u_url = base + "?save=true&filename=" + u_fn_e
 599         d.addCallback(lambda res: self.GET(u_url))
 600         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 601         return d
 602
 603     def test_PUT_FILEURL_named_bad(self):
 604         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 605         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 606                              "400 Bad Request",
 607                              "/file can only be used with GET or HEAD",
 608                              self.PUT, base + "/@@name=/blah.txt", "")
 609         return d
 610
 611     def test_GET_DIRURL_named_bad(self):
 612         base = "/file/%s" % urllib.quote(self._foo_uri)
 613         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 614                              "400 Bad Request",
 615                              "is not a file-cap",
 616                              self.GET, base + "/@@name=/blah.txt")
 617         return d
 618
 619     def test_GET_slash_file_bad(self):
 620         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 621                              "404 Not Found",
 622                              "/file must be followed by a file-cap and a name",
 623                              self.GET, "/file")
 624         return d
 625
 626     def test_GET_unhandled_URI_named(self):
 627         contents, n, newuri = self.makefile(12)
 628         verifier_cap = n.get_verifier().to_string()
 629         base = "/file/%s" % urllib.quote(verifier_cap)
 630         # client.create_node_from_uri() can't handle verify-caps
 631         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 632                              "400 Bad Request",
 633                              "is not a valid file- or directory- cap",
 634                              self.GET, base)
 635         return d
 636
 637     def test_GET_unhandled_URI(self):
 638         contents, n, newuri = self.makefile(12)
 639         verifier_cap = n.get_verifier().to_string()
 640         base = "/uri/%s" % urllib.quote(verifier_cap)
 641         # client.create_node_from_uri() can't handle verify-caps
 642         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 643                              "400 Bad Request",
 644                              "is not a valid file- or directory- cap",
 645                              self.GET, base)
 646         return d
 647
 648     def test_GET_FILE_URI(self):
 649         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 650         d = self.GET(base)
 651         d.addCallback(self.failUnlessIsBarDotTxt)
 652         return d
 653
 654     def test_GET_FILE_URI_badchild(self):
 655         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 656         errmsg = "Files have no children, certainly not named 'boguschild'"
 657         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 658                              "400 Bad Request", errmsg,
 659                              self.GET, base)
 660         return d
 661
 662     def test_PUT_FILE_URI_badchild(self):
 663         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 664         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 665         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 666                              "400 Bad Request", errmsg,
 667                              self.PUT, base, "")
 668         return d
 669
 670     def test_GET_FILEURL_save(self):
 671         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 672         # TODO: look at the headers, expect a Content-Disposition: attachment
 673         # header.
 674         d.addCallback(self.failUnlessIsBarDotTxt)
 675         return d
 676
 677     def test_GET_FILEURL_missing(self):
 678         d = self.GET(self.public_url + "/foo/missing")
 679         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 680         return d
 681
 682     def test_PUT_NEWFILEURL(self):
 683         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 684         # TODO: we lose the response code, so we can't check this
 685         #self.failUnlessEqual(responsecode, 201)
 686         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 687         d.addCallback(lambda res:
 688                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 689                                                       self.NEWFILE_CONTENTS))
 690         return d
 691
 692     def test_PUT_NEWFILEURL_range_bad(self):
 693         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 694         target = self.public_url + "/foo/new.txt"
 695         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 696                              "501 Not Implemented",
 697                              "Content-Range in PUT not yet supported",
 698                              # (and certainly not for immutable files)
 699                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 700                              headers=headers)
 701         d.addCallback(lambda res:
 702                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 703         return d
 704
 705     def test_PUT_NEWFILEURL_mutable(self):
 706         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 707                      self.NEWFILE_CONTENTS)
 708         # TODO: we lose the response code, so we can't check this
 709         #self.failUnlessEqual(responsecode, 201)
 710         def _check_uri(res):
 711             u = uri.from_string_mutable_filenode(res)
 712             self.failUnless(u.is_mutable())
 713             self.failIf(u.is_readonly())
 714             return res
 715         d.addCallback(_check_uri)
 716         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 717         d.addCallback(lambda res:
 718                       self.failUnlessMutableChildContentsAre(self._foo_node,
 719                                                              u"new.txt",
 720                                                              self.NEWFILE_CONTENTS))
 721         return d
 722
 723     def test_PUT_NEWFILEURL_mutable_toobig(self):
 724         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 725                              "413 Request Entity Too Large",
 726                              "SDMF is limited to one segment, and 10001 > 10000",
 727                              self.PUT,
 728                              self.public_url + "/foo/new.txt?mutable=true",
 729                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 730         return d
 731
 732     def test_PUT_NEWFILEURL_replace(self):
 733         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 734         # TODO: we lose the response code, so we can't check this
 735         #self.failUnlessEqual(responsecode, 200)
 736         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 737         d.addCallback(lambda res:
 738                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 739                                                       self.NEWFILE_CONTENTS))
 740         return d
 741
 742     def test_PUT_NEWFILEURL_bad_t(self):
 743         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 744                              "PUT to a file: bad t=bogus",
 745                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 746                              "contents")
 747         return d
 748
 749     def test_PUT_NEWFILEURL_no_replace(self):
 750         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 751                      self.NEWFILE_CONTENTS)
 752         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 753                   "409 Conflict",
 754                   "There was already a child by that name, and you asked me "
 755                   "to not replace it")
 756         return d
 757
 758     def test_PUT_NEWFILEURL_mkdirs(self):
 759         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 760         fn = self._foo_node
 761         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 762         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 763         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 764         d.addCallback(lambda res:
 765                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 766                                                       self.NEWFILE_CONTENTS))
 767         return d
 768
 769     def test_PUT_NEWFILEURL_blocked(self):
 770         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 771                      self.NEWFILE_CONTENTS)
 772         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 773                   "409 Conflict",
 774                   "Unable to create directory 'blockingfile': a file was in the way")
 775         return d
 776
 777     def test_DELETE_FILEURL(self):
 778         d = self.DELETE(self.public_url + "/foo/bar.txt")
 779         d.addCallback(lambda res:
 780                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 781         return d
 782
 783     def test_DELETE_FILEURL_missing(self):
 784         d = self.DELETE(self.public_url + "/foo/missing")
 785         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 786         return d
 787
 788     def test_DELETE_FILEURL_missing2(self):
 789         d = self.DELETE(self.public_url + "/missing/missing")
 790         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 791         return d
 792
 793     def test_GET_FILEURL_json(self):
 794         # twisted.web.http.parse_qs ignores any query args without an '=', so
 795         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 796         # instead. This may make it tricky to emulate the S3 interface
 797         # completely.
 798         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 799         d.addCallback(self.failUnlessIsBarJSON)
 800         return d
 801
 802     def test_GET_FILEURL_json_missing(self):
 803         d = self.GET(self.public_url + "/foo/missing?json")
 804         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 805         return d
 806
 807     def test_GET_FILEURL_uri(self):
 808         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 809         def _check(res):
 810             self.failUnlessEqual(res, self._bar_txt_uri)
 811         d.addCallback(_check)
 812         d.addCallback(lambda res:
 813                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 814         def _check2(res):
 815             # for now, for files, uris and readonly-uris are the same
 816             self.failUnlessEqual(res, self._bar_txt_uri)
 817         d.addCallback(_check2)
 818         return d
 819
 820     def test_GET_FILEURL_badtype(self):
 821         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 822                                   "bad t=bogus",
 823                                   self.GET,
 824                                   self.public_url + "/foo/bar.txt?t=bogus")
 825         return d
 826
 827     def test_GET_FILEURL_uri_missing(self):
 828         d = self.GET(self.public_url + "/foo/missing?t=uri")
 829         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 830         return d
 831
 832     def test_GET_DIRURL(self):
 833         # the addSlash means we get a redirect here
 834         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 835         ROOT = "../../.."
 836         d = self.GET(self.public_url + "/foo", followRedirect=True)
 837         def _check(res):
 838             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 839                             in res, res)
 840             # the FILE reference points to a URI, but it should end in bar.txt
 841             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 842                        (ROOT, urllib.quote(self._bar_txt_uri)))
 843             get_bar = "".join([r'<td>',
 844                                r'<a href="%s">bar.txt</a>' % bar_url,
 845                                r'</td>',
 846                                r'\s+<td>FILE</td>',
 847                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 848                                ])
 849             self.failUnless(re.search(get_bar, res), res)
 850             for line in res.split("\n"):
 851                 # find the line that contains the delete button for bar.txt
 852                 if ("form action" in line and
 853                     'value="delete"' in line and
 854                     'value="bar.txt"' in line):
 855                     # the form target should use a relative URL
 856                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 857                     self.failUnless(('action="%s"' % foo_url) in line, line)
 858                     # and the when_done= should too
 859                     #done_url = urllib.quote(???)
 860                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 861                     #                in line, line)
 862                     break
 863             else:
 864                 self.fail("unable to find delete-bar.txt line", res)
 865
 866             # the DIR reference just points to a URI
 867             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 868             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 869                        + r'\s+<td>DIR</td>')
 870             self.failUnless(re.search(get_sub, res), res)
 871         d.addCallback(_check)
 872
 873         # look at a directory which is readonly
 874         d.addCallback(lambda res:
 875                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 876         def _check2(res):
 877             self.failUnless("(readonly)" in res, res)
 878             self.failIf("Upload a file" in res, res)
 879         d.addCallback(_check2)
 880
 881         # and at a directory that contains a readonly directory
 882         d.addCallback(lambda res:
 883                       self.GET(self.public_url, followRedirect=True))
 884         def _check3(res):
 885             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 886                                       '</td>\s+<td>DIR-RO</td>', res))
 887         d.addCallback(_check3)
 888
 889         return d
 890
 891     def test_GET_DIRURL_badtype(self):
 892         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 893                                   400, "Bad Request",
 894                                   "bad t=bogus",
 895                                   self.GET,
 896                                   self.public_url + "/foo?t=bogus")
 897         return d
 898
 899     def test_GET_DIRURL_json(self):
 900         d = self.GET(self.public_url + "/foo?t=json")
 901         d.addCallback(self.failUnlessIsFooJSON)
 902         return d
 903
 904
 905     def test_POST_DIRURL_manifest_no_ophandle(self):
 906         d = self.shouldFail2(error.Error,
 907                              "test_POST_DIRURL_manifest_no_ophandle",
 908                              "400 Bad Request",
 909                              "slow operation requires ophandle=",
 910                              self.POST, self.public_url, t="start-manifest")
 911         return d
 912
 913     def test_POST_DIRURL_manifest(self):
 914         d = defer.succeed(None)
 915         def getman(ignored, output):
 916             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 917                           followRedirect=True)
 918             d.addCallback(self.wait_for_operation, "125")
 919             d.addCallback(self.get_operation_results, "125", output)
 920             return d
 921         d.addCallback(getman, None)
 922         def _got_html(manifest):
 923             self.failUnless("Manifest of SI=" in manifest)
 924             self.failUnless("<td>sub</td>" in manifest)
 925             self.failUnless(self._sub_uri in manifest)
 926             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 927         d.addCallback(_got_html)
 928
 929         # both t=status and unadorned GET should be identical
 930         d.addCallback(lambda res: self.GET("/operations/125"))
 931         d.addCallback(_got_html)
 932
 933         d.addCallback(getman, "html")
 934         d.addCallback(_got_html)
 935         d.addCallback(getman, "text")
 936         def _got_text(manifest):
 937             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 938             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 939         d.addCallback(_got_text)
 940         d.addCallback(getman, "JSON")
 941         def _got_json(manifest):
 942             data = manifest["manifest"]
 943             got = {}
 944             for (path_list, cap) in data:
 945                 got[tuple(path_list)] = cap
 946             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 947             self.failUnless((u"sub",u"baz.txt") in got)
 948         d.addCallback(_got_json)
 949         return d
 950
 951     def test_POST_DIRURL_deepsize_no_ophandle(self):
 952         d = self.shouldFail2(error.Error,
 953                              "test_POST_DIRURL_deepsize_no_ophandle",
 954                              "400 Bad Request",
 955                              "slow operation requires ophandle=",
 956                              self.POST, self.public_url, t="start-deep-size")
 957         return d
 958
 959     def test_POST_DIRURL_deepsize(self):
 960         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 961                       followRedirect=True)
 962         d.addCallback(self.wait_for_operation, "126")
 963         d.addCallback(self.get_operation_results, "126", "json")
 964         def _got_json(data):
 965             self.failUnlessEqual(data["finished"], True)
 966             size = data["size"]
 967             self.failUnless(size > 1000)
 968         d.addCallback(_got_json)
 969         d.addCallback(self.get_operation_results, "126", "text")
 970         def _got_text(res):
 971             mo = re.search(r'^size: (\d+)$', res, re.M)
 972             self.failUnless(mo, res)
 973             size = int(mo.group(1))
 974             # with directories, the size varies.
 975             self.failUnless(size > 1000)
 976         d.addCallback(_got_text)
 977         return d
 978
 979     def test_POST_DIRURL_deepstats_no_ophandle(self):
 980         d = self.shouldFail2(error.Error,
 981                              "test_POST_DIRURL_deepstats_no_ophandle",
 982                              "400 Bad Request",
 983                              "slow operation requires ophandle=",
 984                              self.POST, self.public_url, t="start-deep-stats")
 985         return d
 986
 987     def test_POST_DIRURL_deepstats(self):
 988         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 989                       followRedirect=True)
 990         d.addCallback(self.wait_for_operation, "127")
 991         d.addCallback(self.get_operation_results, "127", "json")
 992         def _got_json(stats):
 993             expected = {"count-immutable-files": 3,
 994                         "count-mutable-files": 0,
 995                         "count-literal-files": 0,
 996                         "count-files": 3,
 997                         "count-directories": 3,
 998                         "size-immutable-files": 57,
 999                         "size-literal-files": 0,
1000                         #"size-directories": 1912, # varies
1001                         #"largest-directory": 1590,
1002                         "largest-directory-children": 5,
1003                         "largest-immutable-file": 19,
1004                         }
1005             for k,v in expected.iteritems():
1006                 self.failUnlessEqual(stats[k], v,
1007                                      "stats[%s] was %s, not %s" %
1008                                      (k, stats[k], v))
1009             self.failUnlessEqual(stats["size-files-histogram"],
1010                                  [ [11, 31, 3] ])
1011         d.addCallback(_got_json)
1012         return d
1013
1014     def test_GET_DIRURL_uri(self):
1015         d = self.GET(self.public_url + "/foo?t=uri")
1016         def _check(res):
1017             self.failUnlessEqual(res, self._foo_uri)
1018         d.addCallback(_check)
1019         return d
1020
1021     def test_GET_DIRURL_readonly_uri(self):
1022         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1023         def _check(res):
1024             self.failUnlessEqual(res, self._foo_readonly_uri)
1025         d.addCallback(_check)
1026         return d
1027
1028     def test_PUT_NEWDIRURL(self):
1029         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1030         d.addCallback(lambda res:
1031                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1032         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1033         d.addCallback(self.failUnlessNodeKeysAre, [])
1034         return d
1035
1036     def test_PUT_NEWDIRURL_exists(self):
1037         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1038         d.addCallback(lambda res:
1039                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1040         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1041         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1042         return d
1043
1044     def test_PUT_NEWDIRURL_blocked(self):
1045         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1046                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1047                              self.PUT,
1048                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1049         d.addCallback(lambda res:
1050                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1051         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1052         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1053         return d
1054
1055     def test_PUT_NEWDIRURL_mkdir_p(self):
1056         d = defer.succeed(None)
1057         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1058         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1059         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1060         def mkdir_p(mkpnode):
1061             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1062             d = self.POST(url)
1063             def made_subsub(ssuri):
1064                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1065                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1066                 d = self.POST(url)
1067                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1068                 return d
1069             d.addCallback(made_subsub)
1070             return d
1071         d.addCallback(mkdir_p)
1072         return d
1073
1074     def test_PUT_NEWDIRURL_mkdirs(self):
1075         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1076         d.addCallback(lambda res:
1077                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1078         d.addCallback(lambda res:
1079                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1080         d.addCallback(lambda res:
1081                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1082         d.addCallback(self.failUnlessNodeKeysAre, [])
1083         return d
1084
1085     def test_DELETE_DIRURL(self):
1086         d = self.DELETE(self.public_url + "/foo")
1087         d.addCallback(lambda res:
1088                       self.failIfNodeHasChild(self.public_root, u"foo"))
1089         return d
1090
1091     def test_DELETE_DIRURL_missing(self):
1092         d = self.DELETE(self.public_url + "/foo/missing")
1093         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1094         d.addCallback(lambda res:
1095                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1096         return d
1097
1098     def test_DELETE_DIRURL_missing2(self):
1099         d = self.DELETE(self.public_url + "/missing")
1100         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1101         return d
1102
1103     def dump_root(self):
1104         print "NODEWALK"
1105         w = webish.DirnodeWalkerMixin()
1106         def visitor(childpath, childnode, metadata):
1107             print childpath
1108         d = w.walk(self.public_root, visitor)
1109         return d
1110
1111     def failUnlessNodeKeysAre(self, node, expected_keys):
1112         for k in expected_keys:
1113             assert isinstance(k, unicode)
1114         d = node.list()
1115         def _check(children):
1116             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1117         d.addCallback(_check)
1118         return d
1119     def failUnlessNodeHasChild(self, node, name):
1120         assert isinstance(name, unicode)
1121         d = node.list()
1122         def _check(children):
1123             self.failUnless(name in children)
1124         d.addCallback(_check)
1125         return d
1126     def failIfNodeHasChild(self, node, name):
1127         assert isinstance(name, unicode)
1128         d = node.list()
1129         def _check(children):
1130             self.failIf(name in children)
1131         d.addCallback(_check)
1132         return d
1133
1134     def failUnlessChildContentsAre(self, node, name, expected_contents):
1135         assert isinstance(name, unicode)
1136         d = node.get_child_at_path(name)
1137         d.addCallback(lambda node: node.download_to_data())
1138         def _check(contents):
1139             self.failUnlessEqual(contents, expected_contents)
1140         d.addCallback(_check)
1141         return d
1142
1143     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1144         assert isinstance(name, unicode)
1145         d = node.get_child_at_path(name)
1146         d.addCallback(lambda node: node.download_best_version())
1147         def _check(contents):
1148             self.failUnlessEqual(contents, expected_contents)
1149         d.addCallback(_check)
1150         return d
1151
1152     def failUnlessChildURIIs(self, node, name, expected_uri):
1153         assert isinstance(name, unicode)
1154         d = node.get_child_at_path(name)
1155         def _check(child):
1156             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1157         d.addCallback(_check)
1158         return d
1159
1160     def failUnlessURIMatchesChild(self, got_uri, node, name):
1161         assert isinstance(name, unicode)
1162         d = node.get_child_at_path(name)
1163         def _check(child):
1164             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1165         d.addCallback(_check)
1166         return d
1167
1168     def failUnlessCHKURIHasContents(self, got_uri, contents):
1169         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1170
1171     def test_POST_upload(self):
1172         d = self.POST(self.public_url + "/foo", t="upload",
1173                       file=("new.txt", self.NEWFILE_CONTENTS))
1174         fn = self._foo_node
1175         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1176         d.addCallback(lambda res:
1177                       self.failUnlessChildContentsAre(fn, u"new.txt",
1178                                                       self.NEWFILE_CONTENTS))
1179         return d
1180
1181     def test_POST_upload_unicode(self):
1182         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1183         d = self.POST(self.public_url + "/foo", t="upload",
1184                       file=(filename, self.NEWFILE_CONTENTS))
1185         fn = self._foo_node
1186         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1187         d.addCallback(lambda res:
1188                       self.failUnlessChildContentsAre(fn, filename,
1189                                                       self.NEWFILE_CONTENTS))
1190         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1191         d.addCallback(lambda res: self.GET(target_url))
1192         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1193                                                             self.NEWFILE_CONTENTS,
1194                                                             contents))
1195         return d
1196
1197     def test_POST_upload_unicode_named(self):
1198         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1199         d = self.POST(self.public_url + "/foo", t="upload",
1200                       name=filename,
1201                       file=("overridden", self.NEWFILE_CONTENTS))
1202         fn = self._foo_node
1203         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1204         d.addCallback(lambda res:
1205                       self.failUnlessChildContentsAre(fn, filename,
1206                                                       self.NEWFILE_CONTENTS))
1207         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1208         d.addCallback(lambda res: self.GET(target_url))
1209         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1210                                                             self.NEWFILE_CONTENTS,
1211                                                             contents))
1212         return d
1213
1214     def test_POST_upload_no_link(self):
1215         d = self.POST("/uri", t="upload",
1216                       file=("new.txt", self.NEWFILE_CONTENTS))
1217         def _check_upload_results(page):
1218             # this should be a page which describes the results of the upload
1219             # that just finished.
1220             self.failUnless("Upload Results:" in page)
1221             self.failUnless("URI:" in page)
1222             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1223             mo = uri_re.search(page)
1224             self.failUnless(mo, page)
1225             new_uri = mo.group(1)
1226             return new_uri
1227         d.addCallback(_check_upload_results)
1228         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1229         return d
1230
1231     def test_POST_upload_no_link_whendone(self):
1232         d = self.POST("/uri", t="upload", when_done="/",
1233                       file=("new.txt", self.NEWFILE_CONTENTS))
1234         d.addBoth(self.shouldRedirect, "/")
1235         return d
1236
1237     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1238         d = defer.maybeDeferred(callable, *args, **kwargs)
1239         def done(res):
1240             if isinstance(res, failure.Failure):
1241                 res.trap(error.PageRedirect)
1242                 statuscode = res.value.status
1243                 target = res.value.location
1244                 return checker(statuscode, target)
1245             self.fail("%s: callable was supposed to redirect, not return '%s'"
1246                       % (which, res))
1247         d.addBoth(done)
1248         return d
1249
1250     def test_POST_upload_no_link_whendone_results(self):
1251         def check(statuscode, target):
1252             self.failUnlessEqual(statuscode, str(http.FOUND))
1253             self.failUnless(target.startswith(self.webish_url), target)
1254             return client.getPage(target, method="GET")
1255         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1256                                  check,
1257                                  self.POST, "/uri", t="upload",
1258                                  when_done="/uri/%(uri)s",
1259                                  file=("new.txt", self.NEWFILE_CONTENTS))
1260         d.addCallback(lambda res:
1261                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1262         return d
1263
1264     def test_POST_upload_no_link_mutable(self):
1265         d = self.POST("/uri", t="upload", mutable="true",
1266                       file=("new.txt", self.NEWFILE_CONTENTS))
1267         def _check(new_uri):
1268             new_uri = new_uri.strip()
1269             self.new_uri = new_uri
1270             u = IURI(new_uri)
1271             self.failUnless(IMutableFileURI.providedBy(u))
1272             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1273             n = self.s.create_node_from_uri(new_uri)
1274             return n.download_best_version()
1275         d.addCallback(_check)
1276         def _check2(data):
1277             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1278             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1279         d.addCallback(_check2)
1280         def _check3(data):
1281             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1282             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1283         d.addCallback(_check3)
1284         def _check4(data):
1285             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1286         d.addCallback(_check4)
1287         return d
1288
1289     def test_POST_upload_no_link_mutable_toobig(self):
1290         d = self.shouldFail2(error.Error,
1291                              "test_POST_upload_no_link_mutable_toobig",
1292                              "413 Request Entity Too Large",
1293                              "SDMF is limited to one segment, and 10001 > 10000",
1294                              self.POST,
1295                              "/uri", t="upload", mutable="true",
1296                              file=("new.txt",
1297                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1298         return d
1299
1300     def test_POST_upload_mutable(self):
1301         # this creates a mutable file
1302         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1303                       file=("new.txt", self.NEWFILE_CONTENTS))
1304         fn = self._foo_node
1305         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1306         d.addCallback(lambda res:
1307                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1308                                                              self.NEWFILE_CONTENTS))
1309         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1310         def _got(newnode):
1311             self.failUnless(IMutableFileNode.providedBy(newnode))
1312             self.failUnless(newnode.is_mutable())
1313             self.failIf(newnode.is_readonly())
1314             self._mutable_node = newnode
1315             self._mutable_uri = newnode.get_uri()
1316         d.addCallback(_got)
1317
1318         # now upload it again and make sure that the URI doesn't change
1319         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1320         d.addCallback(lambda res:
1321                       self.POST(self.public_url + "/foo", t="upload",
1322                                 mutable="true",
1323                                 file=("new.txt", NEWER_CONTENTS)))
1324         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1325         d.addCallback(lambda res:
1326                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1327                                                              NEWER_CONTENTS))
1328         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1329         def _got2(newnode):
1330             self.failUnless(IMutableFileNode.providedBy(newnode))
1331             self.failUnless(newnode.is_mutable())
1332             self.failIf(newnode.is_readonly())
1333             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1334         d.addCallback(_got2)
1335
1336         # upload a second time, using PUT instead of POST
1337         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1338         d.addCallback(lambda res:
1339                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1340         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1341         d.addCallback(lambda res:
1342                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1343                                                              NEW2_CONTENTS))
1344
1345         # finally list the directory, since mutable files are displayed
1346         # slightly differently
1347
1348         d.addCallback(lambda res:
1349                       self.GET(self.public_url + "/foo/",
1350                                followRedirect=True))
1351         def _check_page(res):
1352             # TODO: assert more about the contents
1353             self.failUnless("SSK" in res)
1354             return res
1355         d.addCallback(_check_page)
1356
1357         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1358         def _got3(newnode):
1359             self.failUnless(IMutableFileNode.providedBy(newnode))
1360             self.failUnless(newnode.is_mutable())
1361             self.failIf(newnode.is_readonly())
1362             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1363         d.addCallback(_got3)
1364
1365         # look at the JSON form of the enclosing directory
1366         d.addCallback(lambda res:
1367                       self.GET(self.public_url + "/foo/?t=json",
1368                                followRedirect=True))
1369         def _check_page_json(res):
1370             parsed = simplejson.loads(res)
1371             self.failUnlessEqual(parsed[0], "dirnode")
1372             children = dict( [(unicode(name),value)
1373                               for (name,value)
1374                               in parsed[1]["children"].iteritems()] )
1375             self.failUnless("new.txt" in children)
1376             new_json = children["new.txt"]
1377             self.failUnlessEqual(new_json[0], "filenode")
1378             self.failUnless(new_json[1]["mutable"])
1379             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1380             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1381             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1382         d.addCallback(_check_page_json)
1383
1384         # and the JSON form of the file
1385         d.addCallback(lambda res:
1386                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1387         def _check_file_json(res):
1388             parsed = simplejson.loads(res)
1389             self.failUnlessEqual(parsed[0], "filenode")
1390             self.failUnless(parsed[1]["mutable"])
1391             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1392             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1393             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1394         d.addCallback(_check_file_json)
1395
1396         # and look at t=uri and t=readonly-uri
1397         d.addCallback(lambda res:
1398                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1399         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1400         d.addCallback(lambda res:
1401                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1402         def _check_ro_uri(res):
1403             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1404             self.failUnlessEqual(res, ro_uri)
1405         d.addCallback(_check_ro_uri)
1406
1407         # make sure we can get to it from /uri/URI
1408         d.addCallback(lambda res:
1409                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1410         d.addCallback(lambda res:
1411                       self.failUnlessEqual(res, NEW2_CONTENTS))
1412
1413         # and that HEAD computes the size correctly
1414         d.addCallback(lambda res:
1415                       self.HEAD(self.public_url + "/foo/new.txt",
1416                                 return_response=True))
1417         def _got_headers((res, status, headers)):
1418             self.failUnlessEqual(res, "")
1419             self.failUnlessEqual(headers["content-length"][0],
1420                                  str(len(NEW2_CONTENTS)))
1421             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1422         d.addCallback(_got_headers)
1423
1424         # make sure that size errors are displayed correctly for overwrite
1425         d.addCallback(lambda res:
1426                       self.shouldFail2(error.Error,
1427                                        "test_POST_upload_mutable-toobig",
1428                                        "413 Request Entity Too Large",
1429                                        "SDMF is limited to one segment, and 10001 > 10000",
1430                                        self.POST,
1431                                        self.public_url + "/foo", t="upload",
1432                                        mutable="true",
1433                                        file=("new.txt",
1434                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1435                                        ))
1436
1437         d.addErrback(self.dump_error)
1438         return d
1439
1440     def test_POST_upload_mutable_toobig(self):
1441         d = self.shouldFail2(error.Error,
1442                              "test_POST_upload_no_link_mutable_toobig",
1443                              "413 Request Entity Too Large",
1444                              "SDMF is limited to one segment, and 10001 > 10000",
1445                              self.POST,
1446                              self.public_url + "/foo",
1447                              t="upload", mutable="true",
1448                              file=("new.txt",
1449                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1450         return d
1451
1452     def dump_error(self, f):
1453         # if the web server returns an error code (like 400 Bad Request),
1454         # web.client.getPage puts the HTTP response body into the .response
1455         # attribute of the exception object that it gives back. It does not
1456         # appear in the Failure's repr(), so the ERROR that trial displays
1457         # will be rather terse and unhelpful. addErrback this method to the
1458         # end of your chain to get more information out of these errors.
1459         if f.check(error.Error):
1460             print "web.error.Error:"
1461             print f
1462             print f.value.response
1463         return f
1464
1465     def test_POST_upload_replace(self):
1466         d = self.POST(self.public_url + "/foo", t="upload",
1467                       file=("bar.txt", self.NEWFILE_CONTENTS))
1468         fn = self._foo_node
1469         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1470         d.addCallback(lambda res:
1471                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1472                                                       self.NEWFILE_CONTENTS))
1473         return d
1474
1475     def test_POST_upload_no_replace_ok(self):
1476         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1477                       file=("new.txt", self.NEWFILE_CONTENTS))
1478         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1479         d.addCallback(lambda res: self.failUnlessEqual(res,
1480                                                        self.NEWFILE_CONTENTS))
1481         return d
1482
1483     def test_POST_upload_no_replace_queryarg(self):
1484         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1485                       file=("bar.txt", self.NEWFILE_CONTENTS))
1486         d.addBoth(self.shouldFail, error.Error,
1487                   "POST_upload_no_replace_queryarg",
1488                   "409 Conflict",
1489                   "There was already a child by that name, and you asked me "
1490                   "to not replace it")
1491         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1492         d.addCallback(self.failUnlessIsBarDotTxt)
1493         return d
1494
1495     def test_POST_upload_no_replace_field(self):
1496         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1497                       file=("bar.txt", self.NEWFILE_CONTENTS))
1498         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1499                   "409 Conflict",
1500                   "There was already a child by that name, and you asked me "
1501                   "to not replace it")
1502         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1503         d.addCallback(self.failUnlessIsBarDotTxt)
1504         return d
1505
1506     def test_POST_upload_whendone(self):
1507         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1508                       file=("new.txt", self.NEWFILE_CONTENTS))
1509         d.addBoth(self.shouldRedirect, "/THERE")
1510         fn = self._foo_node
1511         d.addCallback(lambda res:
1512                       self.failUnlessChildContentsAre(fn, u"new.txt",
1513                                                       self.NEWFILE_CONTENTS))
1514         return d
1515
1516     def test_POST_upload_named(self):
1517         fn = self._foo_node
1518         d = self.POST(self.public_url + "/foo", t="upload",
1519                       name="new.txt", file=self.NEWFILE_CONTENTS)
1520         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1521         d.addCallback(lambda res:
1522                       self.failUnlessChildContentsAre(fn, u"new.txt",
1523                                                       self.NEWFILE_CONTENTS))
1524         return d
1525
1526     def test_POST_upload_named_badfilename(self):
1527         d = self.POST(self.public_url + "/foo", t="upload",
1528                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1529         d.addBoth(self.shouldFail, error.Error,
1530                   "test_POST_upload_named_badfilename",
1531                   "400 Bad Request",
1532                   "name= may not contain a slash",
1533                   )
1534         # make sure that nothing was added
1535         d.addCallback(lambda res:
1536                       self.failUnlessNodeKeysAre(self._foo_node,
1537                                                  [u"bar.txt", u"blockingfile",
1538                                                   u"empty", u"n\u00fc.txt",
1539                                                   u"sub"]))
1540         return d
1541
1542     def test_POST_FILEURL_check(self):
1543         bar_url = self.public_url + "/foo/bar.txt"
1544         d = self.POST(bar_url, t="check")
1545         def _check(res):
1546             self.failUnless("Healthy!" in res)
1547         d.addCallback(_check)
1548         redir_url = "http://allmydata.org/TARGET"
1549         def _check2(statuscode, target):
1550             self.failUnlessEqual(statuscode, str(http.FOUND))
1551             self.failUnlessEqual(target, redir_url)
1552         d.addCallback(lambda res:
1553                       self.shouldRedirect2("test_POST_FILEURL_check",
1554                                            _check2,
1555                                            self.POST, bar_url,
1556                                            t="check",
1557                                            when_done=redir_url))
1558         d.addCallback(lambda res:
1559                       self.POST(bar_url, t="check", return_to=redir_url))
1560         def _check3(res):
1561             self.failUnless("Healthy!" in res)
1562             self.failUnless("Return to parent directory" in res)
1563             self.failUnless(redir_url in res)
1564         d.addCallback(_check3)
1565
1566         d.addCallback(lambda res:
1567                       self.POST(bar_url, t="check", output="JSON"))
1568         def _check_json(res):
1569             data = simplejson.loads(res)
1570             self.failUnless("storage-index" in data)
1571             self.failUnless(data["results"]["healthy"])
1572         d.addCallback(_check_json)
1573
1574         return d
1575
1576     def test_POST_FILEURL_check_and_repair(self):
1577         bar_url = self.public_url + "/foo/bar.txt"
1578         d = self.POST(bar_url, t="check", repair="true")
1579         def _check(res):
1580             self.failUnless("Healthy!" in res)
1581         d.addCallback(_check)
1582         redir_url = "http://allmydata.org/TARGET"
1583         def _check2(statuscode, target):
1584             self.failUnlessEqual(statuscode, str(http.FOUND))
1585             self.failUnlessEqual(target, redir_url)
1586         d.addCallback(lambda res:
1587                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1588                                            _check2,
1589                                            self.POST, bar_url,
1590                                            t="check", repair="true",
1591                                            when_done=redir_url))
1592         d.addCallback(lambda res:
1593                       self.POST(bar_url, t="check", return_to=redir_url))
1594         def _check3(res):
1595             self.failUnless("Healthy!" in res)
1596             self.failUnless("Return to parent directory" in res)
1597             self.failUnless(redir_url in res)
1598         d.addCallback(_check3)
1599         return d
1600
1601     def test_POST_DIRURL_check(self):
1602         foo_url = self.public_url + "/foo/"
1603         d = self.POST(foo_url, t="check")
1604         def _check(res):
1605             self.failUnless("Healthy!" in res)
1606         d.addCallback(_check)
1607         redir_url = "http://allmydata.org/TARGET"
1608         def _check2(statuscode, target):
1609             self.failUnlessEqual(statuscode, str(http.FOUND))
1610             self.failUnlessEqual(target, redir_url)
1611         d.addCallback(lambda res:
1612                       self.shouldRedirect2("test_POST_DIRURL_check",
1613                                            _check2,
1614                                            self.POST, foo_url,
1615                                            t="check",
1616                                            when_done=redir_url))
1617         d.addCallback(lambda res:
1618                       self.POST(foo_url, t="check", return_to=redir_url))
1619         def _check3(res):
1620             self.failUnless("Healthy!" in res)
1621             self.failUnless("Return to parent directory" in res)
1622             self.failUnless(redir_url in res)
1623         d.addCallback(_check3)
1624
1625         d.addCallback(lambda res:
1626                       self.POST(foo_url, t="check", output="JSON"))
1627         def _check_json(res):
1628             data = simplejson.loads(res)
1629             self.failUnless("storage-index" in data)
1630             self.failUnless(data["results"]["healthy"])
1631         d.addCallback(_check_json)
1632
1633         return d
1634
1635     def test_POST_DIRURL_check_and_repair(self):
1636         foo_url = self.public_url + "/foo/"
1637         d = self.POST(foo_url, t="check", repair="true")
1638         def _check(res):
1639             self.failUnless("Healthy!" in res)
1640         d.addCallback(_check)
1641         redir_url = "http://allmydata.org/TARGET"
1642         def _check2(statuscode, target):
1643             self.failUnlessEqual(statuscode, str(http.FOUND))
1644             self.failUnlessEqual(target, redir_url)
1645         d.addCallback(lambda res:
1646                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1647                                            _check2,
1648                                            self.POST, foo_url,
1649                                            t="check", repair="true",
1650                                            when_done=redir_url))
1651         d.addCallback(lambda res:
1652                       self.POST(foo_url, t="check", return_to=redir_url))
1653         def _check3(res):
1654             self.failUnless("Healthy!" in res)
1655             self.failUnless("Return to parent directory" in res)
1656             self.failUnless(redir_url in res)
1657         d.addCallback(_check3)
1658         return d
1659
1660     def wait_for_operation(self, ignored, ophandle):
1661         url = "/operations/" + ophandle
1662         url += "?t=status&output=JSON"
1663         d = self.GET(url)
1664         def _got(res):
1665             data = simplejson.loads(res)
1666             if not data["finished"]:
1667                 d = self.stall(delay=1.0)
1668                 d.addCallback(self.wait_for_operation, ophandle)
1669                 return d
1670             return data
1671         d.addCallback(_got)
1672         return d
1673
1674     def get_operation_results(self, ignored, ophandle, output=None):
1675         url = "/operations/" + ophandle
1676         url += "?t=status"
1677         if output:
1678             url += "&output=" + output
1679         d = self.GET(url)
1680         def _got(res):
1681             if output and output.lower() == "json":
1682                 return simplejson.loads(res)
1683             return res
1684         d.addCallback(_got)
1685         return d
1686
1687     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1688         d = self.shouldFail2(error.Error,
1689                              "test_POST_DIRURL_deepcheck_no_ophandle",
1690                              "400 Bad Request",
1691                              "slow operation requires ophandle=",
1692                              self.POST, self.public_url, t="start-deep-check")
1693         return d
1694
1695     def test_POST_DIRURL_deepcheck(self):
1696         def _check_redirect(statuscode, target):
1697             self.failUnlessEqual(statuscode, str(http.FOUND))
1698             self.failUnless(target.endswith("/operations/123"))
1699         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1700                                  self.POST, self.public_url,
1701                                  t="start-deep-check", ophandle="123")
1702         d.addCallback(self.wait_for_operation, "123")
1703         def _check_json(data):
1704             self.failUnlessEqual(data["finished"], True)
1705             self.failUnlessEqual(data["count-objects-checked"], 8)
1706             self.failUnlessEqual(data["count-objects-healthy"], 8)
1707         d.addCallback(_check_json)
1708         d.addCallback(self.get_operation_results, "123", "html")
1709         def _check_html(res):
1710             self.failUnless("Objects Checked: <span>8</span>" in res)
1711             self.failUnless("Objects Healthy: <span>8</span>" in res)
1712         d.addCallback(_check_html)
1713
1714         d.addCallback(lambda res:
1715                       self.GET("/operations/123/"))
1716         d.addCallback(_check_html) # should be the same as without the slash
1717
1718         d.addCallback(lambda res:
1719                       self.shouldFail2(error.Error, "one", "404 Not Found",
1720                                        "No detailed results for SI bogus",
1721                                        self.GET, "/operations/123/bogus"))
1722
1723         foo_si = self._foo_node.get_storage_index()
1724         foo_si_s = base32.b2a(foo_si)
1725         d.addCallback(lambda res:
1726                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1727         def _check_foo_json(res):
1728             data = simplejson.loads(res)
1729             self.failUnlessEqual(data["storage-index"], foo_si_s)
1730             self.failUnless(data["results"]["healthy"])
1731         d.addCallback(_check_foo_json)
1732         return d
1733
1734     def test_POST_DIRURL_deepcheck_and_repair(self):
1735         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1736                       ophandle="124", output="json", followRedirect=True)
1737         d.addCallback(self.wait_for_operation, "124")
1738         def _check_json(data):
1739             self.failUnlessEqual(data["finished"], True)
1740             self.failUnlessEqual(data["count-objects-checked"], 8)
1741             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1742             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1743             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1744             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1745             self.failUnlessEqual(data["count-repairs-successful"], 0)
1746             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1747             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1748             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1749             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1750         d.addCallback(_check_json)
1751         d.addCallback(self.get_operation_results, "124", "html")
1752         def _check_html(res):
1753             self.failUnless("Objects Checked: <span>8</span>" in res)
1754
1755             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1756             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1757             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1758
1759             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1760             self.failUnless("Repairs Successful: <span>0</span>" in res)
1761             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1762
1763             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1764             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1765             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1766         d.addCallback(_check_html)
1767         return d
1768
1769     def test_POST_FILEURL_bad_t(self):
1770         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1771                              "POST to file: bad t=bogus",
1772                              self.POST, self.public_url + "/foo/bar.txt",
1773                              t="bogus")
1774         return d
1775
1776     def test_POST_mkdir(self): # return value?
1777         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1778         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1779         d.addCallback(self.failUnlessNodeKeysAre, [])
1780         return d
1781
1782     def test_POST_mkdir_2(self):
1783         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1784         d.addCallback(lambda res:
1785                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1787         d.addCallback(self.failUnlessNodeKeysAre, [])
1788         return d
1789
1790     def test_POST_mkdirs_2(self):
1791         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1792         d.addCallback(lambda res:
1793                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1794         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1795         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1796         d.addCallback(self.failUnlessNodeKeysAre, [])
1797         return d
1798
1799     def test_POST_mkdir_no_parentdir_noredirect(self):
1800         d = self.POST("/uri?t=mkdir")
1801         def _after_mkdir(res):
1802             uri.NewDirectoryURI.init_from_string(res)
1803         d.addCallback(_after_mkdir)
1804         return d
1805
1806     def test_POST_mkdir_no_parentdir_redirect(self):
1807         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1808         d.addBoth(self.shouldRedirect, None, statuscode='303')
1809         def _check_target(target):
1810             target = urllib.unquote(target)
1811             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1812         d.addCallback(_check_target)
1813         return d
1814
1815     def test_POST_noparent_bad(self):
1816         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1817                                   "/uri accepts only PUT, PUT?t=mkdir, "
1818                                   "POST?t=upload, and POST?t=mkdir",
1819                                   self.POST, "/uri?t=bogus")
1820         return d
1821
1822     def test_welcome_page_mkdir_button(self):
1823         # Fetch the welcome page.
1824         d = self.GET("/")
1825         def _after_get_welcome_page(res):
1826             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1827             mo = MKDIR_BUTTON_RE.search(res)
1828             formaction = mo.group(1)
1829             formt = mo.group(2)
1830             formaname = mo.group(3)
1831             formavalue = mo.group(4)
1832             return (formaction, formt, formaname, formavalue)
1833         d.addCallback(_after_get_welcome_page)
1834         def _after_parse_form(res):
1835             (formaction, formt, formaname, formavalue) = res
1836             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1837         d.addCallback(_after_parse_form)
1838         d.addBoth(self.shouldRedirect, None, statuscode='303')
1839         return d
1840
1841     def test_POST_mkdir_replace(self): # return value?
1842         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1843         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1844         d.addCallback(self.failUnlessNodeKeysAre, [])
1845         return d
1846
1847     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1848         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1849         d.addBoth(self.shouldFail, error.Error,
1850                   "POST_mkdir_no_replace_queryarg",
1851                   "409 Conflict",
1852                   "There was already a child by that name, and you asked me "
1853                   "to not replace it")
1854         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1855         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1856         return d
1857
1858     def test_POST_mkdir_no_replace_field(self): # return value?
1859         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1860                       replace="false")
1861         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1862                   "409 Conflict",
1863                   "There was already a child by that name, and you asked me "
1864                   "to not replace it")
1865         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1866         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1867         return d
1868
1869     def test_POST_mkdir_whendone_field(self):
1870         d = self.POST(self.public_url + "/foo",
1871                       t="mkdir", name="newdir", when_done="/THERE")
1872         d.addBoth(self.shouldRedirect, "/THERE")
1873         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1874         d.addCallback(self.failUnlessNodeKeysAre, [])
1875         return d
1876
1877     def test_POST_mkdir_whendone_queryarg(self):
1878         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1879                       t="mkdir", name="newdir")
1880         d.addBoth(self.shouldRedirect, "/THERE")
1881         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1882         d.addCallback(self.failUnlessNodeKeysAre, [])
1883         return d
1884
1885     def test_POST_bad_t(self):
1886         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1887                              "POST to a directory with bad t=BOGUS",
1888                              self.POST, self.public_url + "/foo", t="BOGUS")
1889         return d
1890
1891     def test_POST_set_children(self):
1892         contents9, n9, newuri9 = self.makefile(9)
1893         contents10, n10, newuri10 = self.makefile(10)
1894         contents11, n11, newuri11 = self.makefile(11)
1895
1896         reqbody = """{
1897                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1898                                                 "size": 0,
1899                                                 "metadata": {
1900                                                   "ctime": 1002777696.7564139,
1901                                                   "mtime": 1002777696.7564139
1902                                                  }
1903                                                } ],
1904                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1905                                                 "size": 1,
1906                                                 "metadata": {
1907                                                   "ctime": 1002777696.7564139,
1908                                                   "mtime": 1002777696.7564139
1909                                                  }
1910                                                } ],
1911                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1912                                                 "size": 2,
1913                                                 "metadata": {
1914                                                   "ctime": 1002777696.7564139,
1915                                                   "mtime": 1002777696.7564139
1916                                                  }
1917                                                } ]
1918                     }""" % (newuri9, newuri10, newuri11)
1919
1920         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1921
1922         d = client.getPage(url, method="POST", postdata=reqbody)
1923         def _then(res):
1924             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1925             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1926             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1927
1928         d.addCallback(_then)
1929         d.addErrback(self.dump_error)
1930         return d
1931
1932     def test_POST_put_uri(self):
1933         contents, n, newuri = self.makefile(8)
1934         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1935         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1936         d.addCallback(lambda res:
1937                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1938                                                       contents))
1939         return d
1940
1941     def test_POST_put_uri_replace(self):
1942         contents, n, newuri = self.makefile(8)
1943         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1944         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1945         d.addCallback(lambda res:
1946                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1947                                                       contents))
1948         return d
1949
1950     def test_POST_put_uri_no_replace_queryarg(self):
1951         contents, n, newuri = self.makefile(8)
1952         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1953                       name="bar.txt", uri=newuri)
1954         d.addBoth(self.shouldFail, error.Error,
1955                   "POST_put_uri_no_replace_queryarg",
1956                   "409 Conflict",
1957                   "There was already a child by that name, and you asked me "
1958                   "to not replace it")
1959         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1960         d.addCallback(self.failUnlessIsBarDotTxt)
1961         return d
1962
1963     def test_POST_put_uri_no_replace_field(self):
1964         contents, n, newuri = self.makefile(8)
1965         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1966                       name="bar.txt", uri=newuri)
1967         d.addBoth(self.shouldFail, error.Error,
1968                   "POST_put_uri_no_replace_field",
1969                   "409 Conflict",
1970                   "There was already a child by that name, and you asked me "
1971                   "to not replace it")
1972         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1973         d.addCallback(self.failUnlessIsBarDotTxt)
1974         return d
1975
1976     def test_POST_delete(self):
1977         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1978         d.addCallback(lambda res: self._foo_node.list())
1979         def _check(children):
1980             self.failIf(u"bar.txt" in children)
1981         d.addCallback(_check)
1982         return d
1983
1984     def test_POST_rename_file(self):
1985         d = self.POST(self.public_url + "/foo", t="rename",
1986                       from_name="bar.txt", to_name='wibble.txt')
1987         d.addCallback(lambda res:
1988                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1989         d.addCallback(lambda res:
1990                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1991         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1992         d.addCallback(self.failUnlessIsBarDotTxt)
1993         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1994         d.addCallback(self.failUnlessIsBarJSON)
1995         return d
1996
1997     def test_POST_rename_file_redundant(self):
1998         d = self.POST(self.public_url + "/foo", t="rename",
1999                       from_name="bar.txt", to_name='bar.txt')
2000         d.addCallback(lambda res:
2001                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2002         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2003         d.addCallback(self.failUnlessIsBarDotTxt)
2004         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2005         d.addCallback(self.failUnlessIsBarJSON)
2006         return d
2007
2008     def test_POST_rename_file_replace(self):
2009         # rename a file and replace a directory with it
2010         d = self.POST(self.public_url + "/foo", t="rename",
2011                       from_name="bar.txt", to_name='empty')
2012         d.addCallback(lambda res:
2013                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2014         d.addCallback(lambda res:
2015                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2016         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2017         d.addCallback(self.failUnlessIsBarDotTxt)
2018         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2019         d.addCallback(self.failUnlessIsBarJSON)
2020         return d
2021
2022     def test_POST_rename_file_no_replace_queryarg(self):
2023         # rename a file and replace a directory with it
2024         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2025                       from_name="bar.txt", to_name='empty')
2026         d.addBoth(self.shouldFail, error.Error,
2027                   "POST_rename_file_no_replace_queryarg",
2028                   "409 Conflict",
2029                   "There was already a child by that name, and you asked me "
2030                   "to not replace it")
2031         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2032         d.addCallback(self.failUnlessIsEmptyJSON)
2033         return d
2034
2035     def test_POST_rename_file_no_replace_field(self):
2036         # rename a file and replace a directory with it
2037         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2038                       from_name="bar.txt", to_name='empty')
2039         d.addBoth(self.shouldFail, error.Error,
2040                   "POST_rename_file_no_replace_field",
2041                   "409 Conflict",
2042                   "There was already a child by that name, and you asked me "
2043                   "to not replace it")
2044         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2045         d.addCallback(self.failUnlessIsEmptyJSON)
2046         return d
2047
2048     def failUnlessIsEmptyJSON(self, res):
2049         data = simplejson.loads(res)
2050         self.failUnlessEqual(data[0], "dirnode", data)
2051         self.failUnlessEqual(len(data[1]["children"]), 0)
2052
2053     def test_POST_rename_file_slash_fail(self):
2054         d = self.POST(self.public_url + "/foo", t="rename",
2055                       from_name="bar.txt", to_name='kirk/spock.txt')
2056         d.addBoth(self.shouldFail, error.Error,
2057                   "test_POST_rename_file_slash_fail",
2058                   "400 Bad Request",
2059                   "to_name= may not contain a slash",
2060                   )
2061         d.addCallback(lambda res:
2062                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2063         return d
2064
2065     def test_POST_rename_dir(self):
2066         d = self.POST(self.public_url, t="rename",
2067                       from_name="foo", to_name='plunk')
2068         d.addCallback(lambda res:
2069                       self.failIfNodeHasChild(self.public_root, u"foo"))
2070         d.addCallback(lambda res:
2071                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2072         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2073         d.addCallback(self.failUnlessIsFooJSON)
2074         return d
2075
2076     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2077         """ If target is not None then the redirection has to go to target.  If
2078         statuscode is not None then the redirection has to be accomplished with
2079         that HTTP status code."""
2080         if not isinstance(res, failure.Failure):
2081             to_where = (target is None) and "somewhere" or ("to " + target)
2082             self.fail("%s: we were expecting to get redirected %s, not get an"
2083                       " actual page: %s" % (which, to_where, res))
2084         res.trap(error.PageRedirect)
2085         if statuscode is not None:
2086             self.failUnlessEqual(res.value.status, statuscode,
2087                                  "%s: not a redirect" % which)
2088         if target is not None:
2089             # the PageRedirect does not seem to capture the uri= query arg
2090             # properly, so we can't check for it.
2091             realtarget = self.webish_url + target
2092             self.failUnlessEqual(res.value.location, realtarget,
2093                                  "%s: wrong target" % which)
2094         return res.value.location
2095
2096     def test_GET_URI_form(self):
2097         base = "/uri?uri=%s" % self._bar_txt_uri
2098         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2099         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2100         d = self.GET(base)
2101         d.addBoth(self.shouldRedirect, targetbase)
2102         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2103         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2104         d.addCallback(lambda res: self.GET(base+"&t=json"))
2105         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2106         d.addCallback(self.log, "about to get file by uri")
2107         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2108         d.addCallback(self.failUnlessIsBarDotTxt)
2109         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2110         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2111                                            followRedirect=True))
2112         d.addCallback(self.failUnlessIsFooJSON)
2113         d.addCallback(self.log, "got dir by uri")
2114
2115         return d
2116
2117     def test_GET_URI_form_bad(self):
2118         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2119                              "400 Bad Request", "GET /uri requires uri=",
2120                              self.GET, "/uri")
2121         return d
2122
2123     def test_GET_rename_form(self):
2124         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2125                      followRedirect=True)
2126         def _check(res):
2127             self.failUnless('name="when_done" value="."' in res, res)
2128             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2129         d.addCallback(_check)
2130         return d
2131
2132     def log(self, res, msg):
2133         #print "MSG: %s  RES: %s" % (msg, res)
2134         log.msg(msg)
2135         return res
2136
2137     def test_GET_URI_URL(self):
2138         base = "/uri/%s" % self._bar_txt_uri
2139         d = self.GET(base)
2140         d.addCallback(self.failUnlessIsBarDotTxt)
2141         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2142         d.addCallback(self.failUnlessIsBarDotTxt)
2143         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2144         d.addCallback(self.failUnlessIsBarDotTxt)
2145         return d
2146
2147     def test_GET_URI_URL_dir(self):
2148         base = "/uri/%s?t=json" % self._foo_uri
2149         d = self.GET(base)
2150         d.addCallback(self.failUnlessIsFooJSON)
2151         return d
2152
2153     def test_GET_URI_URL_missing(self):
2154         base = "/uri/%s" % self._bad_file_uri
2155         d = self.GET(base)
2156         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2157                   http.GONE, response_substring="NotEnoughSharesError")
2158         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2159         # here? we must arrange for a download to fail after target.open()
2160         # has been called, and then inspect the response to see that it is
2161         # shorter than we expected.
2162         return d
2163
2164     def test_PUT_NEWFILEURL_uri(self):
2165         contents, n, new_uri = self.makefile(8)
2166         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2167         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2168         d.addCallback(lambda res:
2169                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2170                                                       contents))
2171         return d
2172
2173     def test_PUT_NEWFILEURL_uri_replace(self):
2174         contents, n, new_uri = self.makefile(8)
2175         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2176         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2177         d.addCallback(lambda res:
2178                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2179                                                       contents))
2180         return d
2181
2182     def test_PUT_NEWFILEURL_uri_no_replace(self):
2183         contents, n, new_uri = self.makefile(8)
2184         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2185         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2186                   "409 Conflict",
2187                   "There was already a child by that name, and you asked me "
2188                   "to not replace it")
2189         return d
2190
2191     def test_PUT_NEWFILE_URI(self):
2192         file_contents = "New file contents here\n"
2193         d = self.PUT("/uri", file_contents)
2194         def _check(uri):
2195             self.failUnless(uri in FakeCHKFileNode.all_contents)
2196             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2197                                  file_contents)
2198             return self.GET("/uri/%s" % uri)
2199         d.addCallback(_check)
2200         def _check2(res):
2201             self.failUnlessEqual(res, file_contents)
2202         d.addCallback(_check2)
2203         return d
2204
2205     def test_PUT_NEWFILE_URI_only_PUT(self):
2206         d = self.PUT("/uri?t=bogus", "")
2207         d.addBoth(self.shouldFail, error.Error,
2208                   "PUT_NEWFILE_URI_only_PUT",
2209                   "400 Bad Request",
2210                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2211         return d
2212
2213     def test_PUT_NEWFILE_URI_mutable(self):
2214         file_contents = "New file contents here\n"
2215         d = self.PUT("/uri?mutable=true", file_contents)
2216         def _check_mutable(uri):
2217             uri = uri.strip()
2218             u = IURI(uri)
2219             self.failUnless(IMutableFileURI.providedBy(u))
2220             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2221             n = self.s.create_node_from_uri(uri)
2222             return n.download_best_version()
2223         d.addCallback(_check_mutable)
2224         def _check2_mutable(data):
2225             self.failUnlessEqual(data, file_contents)
2226         d.addCallback(_check2_mutable)
2227         return d
2228
2229         def _check(uri):
2230             self.failUnless(uri in FakeCHKFileNode.all_contents)
2231             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2232                                  file_contents)
2233             return self.GET("/uri/%s" % uri)
2234         d.addCallback(_check)
2235         def _check2(res):
2236             self.failUnlessEqual(res, file_contents)
2237         d.addCallback(_check2)
2238         return d
2239
2240     def test_PUT_mkdir(self):
2241         d = self.PUT("/uri?t=mkdir", "")
2242         def _check(uri):
2243             n = self.s.create_node_from_uri(uri.strip())
2244             d2 = self.failUnlessNodeKeysAre(n, [])
2245             d2.addCallback(lambda res:
2246                            self.GET("/uri/%s?t=json" % uri))
2247             return d2
2248         d.addCallback(_check)
2249         d.addCallback(self.failUnlessIsEmptyJSON)
2250         return d
2251
2252     def test_POST_check(self):
2253         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2254         def _done(res):
2255             # this returns a string form of the results, which are probably
2256             # None since we're using fake filenodes.
2257             # TODO: verify that the check actually happened, by changing
2258             # FakeCHKFileNode to count how many times .check() has been
2259             # called.
2260             pass
2261         d.addCallback(_done)
2262         return d
2263
2264     def test_bad_method(self):
2265         url = self.webish_url + self.public_url + "/foo/bar.txt"
2266         d = self.shouldHTTPError2("test_bad_method",
2267                                   501, "Not Implemented",
2268                                   "I don't know how to treat a BOGUS request.",
2269                                   client.getPage, url, method="BOGUS")
2270         return d
2271
2272     def test_short_url(self):
2273         url = self.webish_url + "/uri"
2274         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2275                                   "I don't know how to treat a DELETE request.",
2276                                   client.getPage, url, method="DELETE")
2277         return d
2278
2279     def test_ophandle_bad(self):
2280         url = self.webish_url + "/operations/bogus?t=status"
2281         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2282                                   "unknown/expired handle 'bogus'",
2283                                   client.getPage, url)
2284         return d
2285
2286     def test_ophandle_cancel(self):
2287         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2288                       followRedirect=True)
2289         d.addCallback(lambda ignored:
2290                       self.GET("/operations/128?t=status&output=JSON"))
2291         def _check1(res):
2292             data = simplejson.loads(res)
2293             self.failUnless("finished" in data, res)
2294             monitor = self.ws.root.child_operations.handles["128"][0]
2295             d = self.POST("/operations/128?t=cancel&output=JSON")
2296             def _check2(res):
2297                 data = simplejson.loads(res)
2298                 self.failUnless("finished" in data, res)
2299                 # t=cancel causes the handle to be forgotten
2300                 self.failUnless(monitor.is_cancelled())
2301             d.addCallback(_check2)
2302             return d
2303         d.addCallback(_check1)
2304         d.addCallback(lambda ignored:
2305                       self.shouldHTTPError2("test_ophandle_cancel",
2306                                             404, "404 Not Found",
2307                                             "unknown/expired handle '128'",
2308                                             self.GET,
2309                                             "/operations/128?t=status&output=JSON"))
2310         return d
2311
2312     def test_ophandle_retainfor(self):
2313         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2314                       followRedirect=True)
2315         d.addCallback(lambda ignored:
2316                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2317         def _check1(res):
2318             data = simplejson.loads(res)
2319             self.failUnless("finished" in data, res)
2320         d.addCallback(_check1)
2321         # the retain-for=0 will cause the handle to be expired very soon
2322         d.addCallback(self.stall, 2.0)
2323         d.addCallback(lambda ignored:
2324                       self.shouldHTTPError2("test_ophandle_retainfor",
2325                                             404, "404 Not Found",
2326                                             "unknown/expired handle '129'",
2327                                             self.GET,
2328                                             "/operations/129?t=status&output=JSON"))
2329         return d
2330
2331     def test_ophandle_release_after_complete(self):
2332         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2333                       followRedirect=True)
2334         d.addCallback(self.wait_for_operation, "130")
2335         d.addCallback(lambda ignored:
2336                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2337         # the release-after-complete=true will cause the handle to be expired
2338         d.addCallback(lambda ignored:
2339                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2340                                             404, "404 Not Found",
2341                                             "unknown/expired handle '130'",
2342                                             self.GET,
2343                                             "/operations/130?t=status&output=JSON"))
2344         return d
2345
2346     def test_incident(self):
2347         d = self.POST("/report_incident", details="eek")
2348         def _done(res):
2349             self.failUnless("Thank you for your report!" in res, res)
2350         d.addCallback(_done)
2351         return d
2352
2353
2354 class Util(unittest.TestCase):
2355     def test_abbreviate_time(self):
2356         self.failUnlessEqual(common.abbreviate_time(None), "")
2357         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2358         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2359         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2360         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2361
2362     def test_abbreviate_rate(self):
2363         self.failUnlessEqual(common.abbreviate_rate(None), "")
2364         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2365         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2366         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2367
2368     def test_abbreviate_size(self):
2369         self.failUnlessEqual(common.abbreviate_size(None), "")
2370         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2371         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2372         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2373         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2374