src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17 import common_util as testutil
  18
  19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  20 # create a webserver that works against them
  21
  22 class FakeIntroducerClient:
  23     def get_all_connectors(self):
  24         return {}
  25     def get_all_connections_for(self, service_name):
  26         return frozenset()
  27     def get_all_peerids(self):
  28         return frozenset()
  29
  30 class FakeClient(service.MultiService):
  31     nodeid = "fake_nodeid"
  32     nickname = "fake_nickname"
  33     basedir = "fake_basedir"
  34     def get_versions(self):
  35         return {'allmydata': "fake",
  36                 'foolscap': "fake",
  37                 'twisted': "fake",
  38                 'zfec': "fake",
  39                 }
  40     introducer_furl = "None"
  41     introducer_client = FakeIntroducerClient()
  42     _all_upload_status = [upload.UploadStatus()]
  43     _all_download_status = [download.DownloadStatus()]
  44     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  45     _all_publish_statuses = [publish.PublishStatus()]
  46     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  47     convergence = "some random string"
  48
  49     def connected_to_introducer(self):
  50         return False
  51
  52     def get_nickname_for_peerid(self, peerid):
  53         return u"John Doe"
  54
  55     def create_node_from_uri(self, auri):
  56         u = uri.from_string(auri)
  57         if (INewDirectoryURI.providedBy(u)
  58             or IReadonlyNewDirectoryURI.providedBy(u)):
  59             return FakeDirectoryNode(self).init_from_uri(u)
  60         if IFileURI.providedBy(u):
  61             return FakeCHKFileNode(u, self)
  62         assert IMutableFileURI.providedBy(u), u
  63         return FakeMutableFileNode(self).init_from_uri(u)
  64
  65     def create_empty_dirnode(self):
  66         n = FakeDirectoryNode(self)
  67         d = n.create()
  68         d.addCallback(lambda res: n)
  69         return d
  70
  71     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  72     def create_mutable_file(self, contents=""):
  73         n = FakeMutableFileNode(self)
  74         return n.create(contents)
  75
  76     def upload(self, uploadable):
  77         d = uploadable.get_size()
  78         d.addCallback(lambda size: uploadable.read(size))
  79         def _got_data(datav):
  80             data = "".join(datav)
  81             n = create_chk_filenode(self, data)
  82             results = upload.UploadResults()
  83             results.uri = n.get_uri()
  84             return results
  85         d.addCallback(_got_data)
  86         return d
  87
  88     def list_all_upload_statuses(self):
  89         return self._all_upload_status
  90     def list_all_download_statuses(self):
  91         return self._all_download_status
  92     def list_all_mapupdate_statuses(self):
  93         return self._all_mapupdate_statuses
  94     def list_all_publish_statuses(self):
  95         return self._all_publish_statuses
  96     def list_all_retrieve_statuses(self):
  97         return self._all_retrieve_statuses
  98     def list_all_helper_statuses(self):
  99         return []
 100
 101 class MyGetter(client.HTTPPageGetter):
 102     handleStatus_206 = lambda self: self.handleStatus_200()
 103
 104 class HTTPClientHEADFactory(client.HTTPClientFactory):
 105     protocol = MyGetter
 106
 107     def noPage(self, reason):
 108         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 109         # exception when the response to a HEAD request had no body (when in
 110         # fact they are defined to never have a body). This was fixed in
 111         # Twisted-8.0 . To work around this, we catch the
 112         # PartialDownloadError and make it disappear.
 113         if (reason.check(client.PartialDownloadError)
 114             and self.method.upper() == "HEAD"):
 115             self.page("")
 116             return
 117         return client.HTTPClientFactory.noPage(self, reason)
 118
 119 class HTTPClientGETFactory(client.HTTPClientFactory):
 120     protocol = MyGetter
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 128         s.setServiceParent(self.s)
 129         self.webish_port = port = s.listener._port.getHost().port
 130         self.webish_url = "http://localhost:%d" % port
 131
 132         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 133         d = defer.DeferredList(l)
 134         def _then(res):
 135             self.public_root = res[0][1]
 136             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 137             self.public_url = "/uri/" + self.public_root.get_uri()
 138             self.private_root = res[1][1]
 139
 140             foo = res[2][1]
 141             self._foo_node = foo
 142             self._foo_uri = foo.get_uri()
 143             self._foo_readonly_uri = foo.get_readonly_uri()
 144             # NOTE: we ignore the deferred on all set_uri() calls, because we
 145             # know the fake nodes do these synchronously
 146             self.public_root.set_uri(u"foo", foo.get_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 150
 151             foo.set_uri(u"empty", res[3][1].get_uri())
 152             sub_uri = res[4][1].get_uri()
 153             self._sub_uri = sub_uri
 154             foo.set_uri(u"sub", sub_uri)
 155             sub = self.s.create_node_from_uri(sub_uri)
 156
 157             _ign, n, blocking_uri = self.makefile(1)
 158             foo.set_uri(u"blockingfile", blocking_uri)
 159
 160             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 161             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 162             # still think of it as an umlaut
 163             foo.set_uri(unicode_filename, self._bar_txt_uri)
 164
 165             _ign, n, baz_file = self.makefile(2)
 166             sub.set_uri(u"baz.txt", baz_file)
 167
 168             _ign, n, self._bad_file_uri = self.makefile(3)
 169             # this uri should not be downloadable
 170             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 171
 172             rodir = res[5][1]
 173             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(self.s, contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 214
 215     def failUnlessIsFooJSON(self, res):
 216         data = simplejson.loads(res)
 217         self.failUnless(isinstance(data, list))
 218         self.failUnlessEqual(data[0], "dirnode", res)
 219         self.failUnless(isinstance(data[1], dict))
 220         self.failUnless(data[1]["mutable"])
 221         self.failUnless("rw_uri" in data[1]) # mutable
 222         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 223         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 224
 225         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 226         self.failUnlessEqual(kidnames,
 227                              [u"bar.txt", u"blockingfile", u"empty",
 228                               u"n\u00fc.txt", u"sub"])
 229         kids = dict( [(unicode(name),value)
 230                       for (name,value)
 231                       in data[1]["children"].iteritems()] )
 232         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 233         self.failUnless("metadata" in kids[u"sub"][1])
 234         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 235         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 236         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 237         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 238         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 240                              self._bar_txt_metadata["ctime"])
 241         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 242                              self._bar_txt_uri)
 243
 244     def GET(self, urlpath, followRedirect=False, return_response=False,
 245             **kwargs):
 246         # if return_response=True, this fires with (data, statuscode,
 247         # respheaders) instead of just data.
 248         assert not isinstance(urlpath, unicode)
 249         url = self.webish_url + urlpath
 250         factory = HTTPClientGETFactory(url, method="GET",
 251                                        followRedirect=followRedirect, **kwargs)
 252         reactor.connectTCP("localhost", self.webish_port, factory)
 253         d = factory.deferred
 254         def _got_data(data):
 255             return (data, factory.status, factory.response_headers)
 256         if return_response:
 257             d.addCallback(_got_data)
 258         return factory.deferred
 259
 260     def HEAD(self, urlpath, return_response=False, **kwargs):
 261         # this requires some surgery, because twisted.web.client doesn't want
 262         # to give us back the response headers.
 263         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 264         reactor.connectTCP("localhost", self.webish_port, factory)
 265         d = factory.deferred
 266         def _got_data(data):
 267             return (data, factory.status, factory.response_headers)
 268         if return_response:
 269             d.addCallback(_got_data)
 270         return factory.deferred
 271
 272     def PUT(self, urlpath, data, **kwargs):
 273         url = self.webish_url + urlpath
 274         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 275
 276     def DELETE(self, urlpath):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="DELETE")
 279
 280     def POST(self, urlpath, followRedirect=False, **fields):
 281         url = self.webish_url + urlpath
 282         sepbase = "boogabooga"
 283         sep = "--" + sepbase
 284         form = []
 285         form.append(sep)
 286         form.append('Content-Disposition: form-data; name="_charset"')
 287         form.append('')
 288         form.append('UTF-8')
 289         form.append(sep)
 290         for name, value in fields.iteritems():
 291             if isinstance(value, tuple):
 292                 filename, value = value
 293                 form.append('Content-Disposition: form-data; name="%s"; '
 294                             'filename="%s"' % (name, filename.encode("utf-8")))
 295             else:
 296                 form.append('Content-Disposition: form-data; name="%s"' % name)
 297             form.append('')
 298             if isinstance(value, unicode):
 299                 value = value.encode("utf-8")
 300             else:
 301                 value = str(value)
 302             assert isinstance(value, str)
 303             form.append(value)
 304             form.append(sep)
 305         form[-1] += "--"
 306         body = "\r\n".join(form) + "\r\n"
 307         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 308                    }
 309         return client.getPage(url, method="POST", postdata=body,
 310                               headers=headers, followRedirect=followRedirect)
 311
 312     def shouldFail(self, res, expected_failure, which,
 313                    substring=None, response_substring=None):
 314         if isinstance(res, failure.Failure):
 315             res.trap(expected_failure)
 316             if substring:
 317                 self.failUnless(substring in str(res),
 318                                 "substring '%s' not in '%s'"
 319                                 % (substring, str(res)))
 320             if response_substring:
 321                 self.failUnless(response_substring in res.value.response,
 322                                 "response substring '%s' not in '%s'"
 323                                 % (response_substring, res.value.response))
 324         else:
 325             self.fail("%s was supposed to raise %s, not get '%s'" %
 326                       (which, expected_failure, res))
 327
 328     def shouldFail2(self, expected_failure, which, substring,
 329                     response_substring,
 330                     callable, *args, **kwargs):
 331         assert substring is None or isinstance(substring, str)
 332         assert response_substring is None or isinstance(response_substring, str)
 333         d = defer.maybeDeferred(callable, *args, **kwargs)
 334         def done(res):
 335             if isinstance(res, failure.Failure):
 336                 res.trap(expected_failure)
 337                 if substring:
 338                     self.failUnless(substring in str(res),
 339                                     "%s: substring '%s' not in '%s'"
 340                                     % (which, substring, str(res)))
 341                 if response_substring:
 342                     self.failUnless(response_substring in res.value.response,
 343                                     "%s: response substring '%s' not in '%s'"
 344                                     % (which,
 345                                        response_substring, res.value.response))
 346             else:
 347                 self.fail("%s was supposed to raise %s, not get '%s'" %
 348                           (which, expected_failure, res))
 349         d.addBoth(done)
 350         return d
 351
 352     def should404(self, res, which):
 353         if isinstance(res, failure.Failure):
 354             res.trap(error.Error)
 355             self.failUnlessEqual(res.value.status, "404")
 356         else:
 357             self.fail("%s was supposed to Error(404), not get '%s'" %
 358                       (which, res))
 359
 360     def shouldHTTPError(self, res, which, code=None, substring=None,
 361                         response_substring=None):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             if code is not None:
 365                 self.failUnlessEqual(res.value.status, str(code))
 366             if substring:
 367                 self.failUnless(substring in str(res),
 368                                 "substring '%s' not in '%s'"
 369                                 % (substring, str(res)))
 370             if response_substring:
 371                 self.failUnless(response_substring in res.value.response,
 372                                 "response substring '%s' not in '%s'"
 373                                 % (response_substring, res.value.response))
 374         else:
 375             self.fail("%s was supposed to Error(%s), not get '%s'" %
 376                       (which, code, res))
 377
 378     def shouldHTTPError2(self, which,
 379                          code=None, substring=None, response_substring=None,
 380                          callable=None, *args, **kwargs):
 381         assert substring is None or isinstance(substring, str)
 382         assert callable
 383         d = defer.maybeDeferred(callable, *args, **kwargs)
 384         d.addBoth(self.shouldHTTPError, which,
 385                   code, substring, response_substring)
 386         return d
 387
 388
 389 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 390     def test_create(self):
 391         pass
 392
 393     def test_welcome(self):
 394         d = self.GET("/")
 395         def _check(res):
 396             self.failUnless('Welcome To AllMyData' in res)
 397             self.failUnless('Tahoe' in res)
 398
 399             self.s.basedir = 'web/test_welcome'
 400             fileutil.make_dirs("web/test_welcome")
 401             fileutil.make_dirs("web/test_welcome/private")
 402             return self.GET("/")
 403         d.addCallback(_check)
 404         return d
 405
 406     def test_provisioning_math(self):
 407         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 408         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 409         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 410         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 411         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 412
 413     def test_provisioning(self):
 414         d = self.GET("/provisioning/")
 415         def _check(res):
 416             self.failUnless('Tahoe Provisioning Tool' in res)
 417             fields = {'filled': True,
 418                       "num_users": int(50e3),
 419                       "files_per_user": 1000,
 420                       "space_per_user": int(1e9),
 421                       "sharing_ratio": 1.0,
 422                       "encoding_parameters": "3-of-10-5",
 423                       "num_servers": 30,
 424                       "ownership_mode": "A",
 425                       "download_rate": 100,
 426                       "upload_rate": 10,
 427                       "delete_rate": 10,
 428                       "lease_timer": 7,
 429                       }
 430             return self.POST("/provisioning/", **fields)
 431
 432         d.addCallback(_check)
 433         def _check2(res):
 434             self.failUnless('Tahoe Provisioning Tool' in res)
 435             self.failUnless("Share space consumed: 167.01TB" in res)
 436
 437             fields = {'filled': True,
 438                       "num_users": int(50e6),
 439                       "files_per_user": 1000,
 440                       "space_per_user": int(5e9),
 441                       "sharing_ratio": 1.0,
 442                       "encoding_parameters": "25-of-100-50",
 443                       "num_servers": 30000,
 444                       "ownership_mode": "E",
 445                       "drive_failure_model": "U",
 446                       "drive_size": 1000,
 447                       "download_rate": 1000,
 448                       "upload_rate": 100,
 449                       "delete_rate": 100,
 450                       "lease_timer": 7,
 451                       }
 452             return self.POST("/provisioning/", **fields)
 453         d.addCallback(_check2)
 454         def _check3(res):
 455             self.failUnless("Share space consumed: huge!" in res)
 456             fields = {'filled': True}
 457             return self.POST("/provisioning/", **fields)
 458         d.addCallback(_check3)
 459         def _check4(res):
 460             self.failUnless("Share space consumed:" in res)
 461         d.addCallback(_check4)
 462         return d
 463
 464     def test_status(self):
 465         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 466         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 467         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 468         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 469         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 470         d = self.GET("/status", followRedirect=True)
 471         def _check(res):
 472             self.failUnless('Upload and Download Status' in res, res)
 473             self.failUnless('"down-%d"' % dl_num in res, res)
 474             self.failUnless('"up-%d"' % ul_num in res, res)
 475             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 476             self.failUnless('"publish-%d"' % pub_num in res, res)
 477             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 478         d.addCallback(_check)
 479         d.addCallback(lambda res: self.GET("/status/?t=json"))
 480         def _check_json(res):
 481             data = simplejson.loads(res)
 482             self.failUnless(isinstance(data, dict))
 483             active = data["active"]
 484             # TODO: test more. We need a way to fake an active operation
 485             # here.
 486         d.addCallback(_check_json)
 487
 488         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 489         def _check_dl(res):
 490             self.failUnless("File Download Status" in res, res)
 491         d.addCallback(_check_dl)
 492         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 493         def _check_ul(res):
 494             self.failUnless("File Upload Status" in res, res)
 495         d.addCallback(_check_ul)
 496         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 497         def _check_mapupdate(res):
 498             self.failUnless("Mutable File Servermap Update Status" in res, res)
 499         d.addCallback(_check_mapupdate)
 500         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 501         def _check_publish(res):
 502             self.failUnless("Mutable File Publish Status" in res, res)
 503         d.addCallback(_check_publish)
 504         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 505         def _check_retrieve(res):
 506             self.failUnless("Mutable File Retrieve Status" in res, res)
 507         d.addCallback(_check_retrieve)
 508
 509         return d
 510
 511     def test_status_numbers(self):
 512         drrm = status.DownloadResultsRendererMixin()
 513         self.failUnlessEqual(drrm.render_time(None, None), "")
 514         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 515         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 517         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 518         self.failUnlessEqual(drrm.render_rate(None, None), "")
 519         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 521         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 522
 523         urrm = status.UploadResultsRendererMixin()
 524         self.failUnlessEqual(urrm.render_time(None, None), "")
 525         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 526         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 528         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 529         self.failUnlessEqual(urrm.render_rate(None, None), "")
 530         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 532         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 533
 534     def test_GET_FILEURL(self):
 535         d = self.GET(self.public_url + "/foo/bar.txt")
 536         d.addCallback(self.failUnlessIsBarDotTxt)
 537         return d
 538
 539     def test_GET_FILEURL_range(self):
 540         headers = {"range": "bytes=1-10"}
 541         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 542                      return_response=True)
 543         def _got((res, status, headers)):
 544             self.failUnlessEqual(int(status), 206)
 545             self.failUnless(headers.has_key("content-range"))
 546             self.failUnlessEqual(headers["content-range"][0],
 547                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 548             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 549         d.addCallback(_got)
 550         return d
 551
 552     def test_HEAD_FILEURL_range(self):
 553         headers = {"range": "bytes=1-10"}
 554         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 555                      return_response=True)
 556         def _got((res, status, headers)):
 557             self.failUnlessEqual(res, "")
 558             self.failUnlessEqual(int(status), 206)
 559             self.failUnless(headers.has_key("content-range"))
 560             self.failUnlessEqual(headers["content-range"][0],
 561                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 562         d.addCallback(_got)
 563         return d
 564
 565     def test_GET_FILEURL_range_bad(self):
 566         headers = {"range": "BOGUS=fizbop-quarnak"}
 567         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 568                              "400 Bad Request",
 569                              "Syntactically invalid http range header",
 570                              self.GET, self.public_url + "/foo/bar.txt",
 571                              headers=headers)
 572         return d
 573
 574     def test_HEAD_FILEURL(self):
 575         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 576         def _got((res, status, headers)):
 577             self.failUnlessEqual(res, "")
 578             self.failUnlessEqual(headers["content-length"][0],
 579                                  str(len(self.BAR_CONTENTS)))
 580             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 581         d.addCallback(_got)
 582         return d
 583
 584     def test_GET_FILEURL_named(self):
 585         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 586         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 587         d = self.GET(base + "/@@name=/blah.txt")
 588         d.addCallback(self.failUnlessIsBarDotTxt)
 589         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 590         d.addCallback(self.failUnlessIsBarDotTxt)
 591         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 592         d.addCallback(self.failUnlessIsBarDotTxt)
 593         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 594         d.addCallback(self.failUnlessIsBarDotTxt)
 595         save_url = base + "?save=true&filename=blah.txt"
 596         d.addCallback(lambda res: self.GET(save_url))
 597         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 598         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 599         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 600         u_url = base + "?save=true&filename=" + u_fn_e
 601         d.addCallback(lambda res: self.GET(u_url))
 602         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 603         return d
 604
 605     def test_PUT_FILEURL_named_bad(self):
 606         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 607         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 608                              "400 Bad Request",
 609                              "/file can only be used with GET or HEAD",
 610                              self.PUT, base + "/@@name=/blah.txt", "")
 611         return d
 612
 613     def test_GET_DIRURL_named_bad(self):
 614         base = "/file/%s" % urllib.quote(self._foo_uri)
 615         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 616                              "400 Bad Request",
 617                              "is not a file-cap",
 618                              self.GET, base + "/@@name=/blah.txt")
 619         return d
 620
 621     def test_GET_slash_file_bad(self):
 622         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 623                              "404 Not Found",
 624                              "/file must be followed by a file-cap and a name",
 625                              self.GET, "/file")
 626         return d
 627
 628     def test_GET_unhandled_URI_named(self):
 629         contents, n, newuri = self.makefile(12)
 630         verifier_cap = n.get_verifier().to_string()
 631         base = "/file/%s" % urllib.quote(verifier_cap)
 632         # client.create_node_from_uri() can't handle verify-caps
 633         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 634                              "400 Bad Request",
 635                              "is not a valid file- or directory- cap",
 636                              self.GET, base)
 637         return d
 638
 639     def test_GET_unhandled_URI(self):
 640         contents, n, newuri = self.makefile(12)
 641         verifier_cap = n.get_verifier().to_string()
 642         base = "/uri/%s" % urllib.quote(verifier_cap)
 643         # client.create_node_from_uri() can't handle verify-caps
 644         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 645                              "400 Bad Request",
 646                              "is not a valid file- or directory- cap",
 647                              self.GET, base)
 648         return d
 649
 650     def test_GET_FILE_URI(self):
 651         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 652         d = self.GET(base)
 653         d.addCallback(self.failUnlessIsBarDotTxt)
 654         return d
 655
 656     def test_GET_FILE_URI_badchild(self):
 657         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 658         errmsg = "Files have no children, certainly not named 'boguschild'"
 659         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 660                              "400 Bad Request", errmsg,
 661                              self.GET, base)
 662         return d
 663
 664     def test_PUT_FILE_URI_badchild(self):
 665         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 666         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 667         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 668                              "400 Bad Request", errmsg,
 669                              self.PUT, base, "")
 670         return d
 671
 672     def test_GET_FILEURL_save(self):
 673         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 674         # TODO: look at the headers, expect a Content-Disposition: attachment
 675         # header.
 676         d.addCallback(self.failUnlessIsBarDotTxt)
 677         return d
 678
 679     def test_GET_FILEURL_missing(self):
 680         d = self.GET(self.public_url + "/foo/missing")
 681         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 682         return d
 683
 684     def test_PUT_NEWFILEURL(self):
 685         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 686         # TODO: we lose the response code, so we can't check this
 687         #self.failUnlessEqual(responsecode, 201)
 688         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 689         d.addCallback(lambda res:
 690                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 691                                                       self.NEWFILE_CONTENTS))
 692         return d
 693
 694     def test_PUT_NEWFILEURL_range_bad(self):
 695         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 696         target = self.public_url + "/foo/new.txt"
 697         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 698                              "501 Not Implemented",
 699                              "Content-Range in PUT not yet supported",
 700                              # (and certainly not for immutable files)
 701                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 702                              headers=headers)
 703         d.addCallback(lambda res:
 704                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 705         return d
 706
 707     def test_PUT_NEWFILEURL_mutable(self):
 708         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 709                      self.NEWFILE_CONTENTS)
 710         # TODO: we lose the response code, so we can't check this
 711         #self.failUnlessEqual(responsecode, 201)
 712         def _check_uri(res):
 713             u = uri.from_string_mutable_filenode(res)
 714             self.failUnless(u.is_mutable())
 715             self.failIf(u.is_readonly())
 716             return res
 717         d.addCallback(_check_uri)
 718         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 719         d.addCallback(lambda res:
 720                       self.failUnlessMutableChildContentsAre(self._foo_node,
 721                                                              u"new.txt",
 722                                                              self.NEWFILE_CONTENTS))
 723         return d
 724
 725     def test_PUT_NEWFILEURL_mutable_toobig(self):
 726         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 727                              "413 Request Entity Too Large",
 728                              "SDMF is limited to one segment, and 10001 > 10000",
 729                              self.PUT,
 730                              self.public_url + "/foo/new.txt?mutable=true",
 731                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 732         return d
 733
 734     def test_PUT_NEWFILEURL_replace(self):
 735         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 200)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_bad_t(self):
 745         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 746                              "PUT to a file: bad t=bogus",
 747                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 748                              "contents")
 749         return d
 750
 751     def test_PUT_NEWFILEURL_no_replace(self):
 752         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 753                      self.NEWFILE_CONTENTS)
 754         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 755                   "409 Conflict",
 756                   "There was already a child by that name, and you asked me "
 757                   "to not replace it")
 758         return d
 759
 760     def test_PUT_NEWFILEURL_mkdirs(self):
 761         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 762         fn = self._foo_node
 763         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 764         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 765         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_blocked(self):
 772         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 773                      self.NEWFILE_CONTENTS)
 774         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 775                   "409 Conflict",
 776                   "Unable to create directory 'blockingfile': a file was in the way")
 777         return d
 778
 779     def test_DELETE_FILEURL(self):
 780         d = self.DELETE(self.public_url + "/foo/bar.txt")
 781         d.addCallback(lambda res:
 782                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 783         return d
 784
 785     def test_DELETE_FILEURL_missing(self):
 786         d = self.DELETE(self.public_url + "/foo/missing")
 787         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 788         return d
 789
 790     def test_DELETE_FILEURL_missing2(self):
 791         d = self.DELETE(self.public_url + "/missing/missing")
 792         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 793         return d
 794
 795     def test_GET_FILEURL_json(self):
 796         # twisted.web.http.parse_qs ignores any query args without an '=', so
 797         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 798         # instead. This may make it tricky to emulate the S3 interface
 799         # completely.
 800         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 801         d.addCallback(self.failUnlessIsBarJSON)
 802         return d
 803
 804     def test_GET_FILEURL_json_missing(self):
 805         d = self.GET(self.public_url + "/foo/missing?json")
 806         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 807         return d
 808
 809     def test_GET_FILEURL_uri(self):
 810         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 811         def _check(res):
 812             self.failUnlessEqual(res, self._bar_txt_uri)
 813         d.addCallback(_check)
 814         d.addCallback(lambda res:
 815                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 816         def _check2(res):
 817             # for now, for files, uris and readonly-uris are the same
 818             self.failUnlessEqual(res, self._bar_txt_uri)
 819         d.addCallback(_check2)
 820         return d
 821
 822     def test_GET_FILEURL_badtype(self):
 823         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 824                                   "bad t=bogus",
 825                                   self.GET,
 826                                   self.public_url + "/foo/bar.txt?t=bogus")
 827         return d
 828
 829     def test_GET_FILEURL_uri_missing(self):
 830         d = self.GET(self.public_url + "/foo/missing?t=uri")
 831         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 832         return d
 833
 834     def test_GET_DIRURL(self):
 835         # the addSlash means we get a redirect here
 836         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 837         ROOT = "../../.."
 838         d = self.GET(self.public_url + "/foo", followRedirect=True)
 839         def _check(res):
 840             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 841                             in res, res)
 842             # the FILE reference points to a URI, but it should end in bar.txt
 843             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 844                        (ROOT, urllib.quote(self._bar_txt_uri)))
 845             get_bar = "".join([r'<td>',
 846                                r'<a href="%s">bar.txt</a>' % bar_url,
 847                                r'</td>',
 848                                r'\s+<td>FILE</td>',
 849                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 850                                ])
 851             self.failUnless(re.search(get_bar, res), res)
 852             for line in res.split("\n"):
 853                 # find the line that contains the delete button for bar.txt
 854                 if ("form action" in line and
 855                     'value="delete"' in line and
 856                     'value="bar.txt"' in line):
 857                     # the form target should use a relative URL
 858                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 859                     self.failUnless(('action="%s"' % foo_url) in line, line)
 860                     # and the when_done= should too
 861                     #done_url = urllib.quote(???)
 862                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 863                     #                in line, line)
 864                     break
 865             else:
 866                 self.fail("unable to find delete-bar.txt line", res)
 867
 868             # the DIR reference just points to a URI
 869             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 870             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 871                        + r'\s+<td>DIR</td>')
 872             self.failUnless(re.search(get_sub, res), res)
 873         d.addCallback(_check)
 874
 875         # look at a directory which is readonly
 876         d.addCallback(lambda res:
 877                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 878         def _check2(res):
 879             self.failUnless("(readonly)" in res, res)
 880             self.failIf("Upload a file" in res, res)
 881         d.addCallback(_check2)
 882
 883         # and at a directory that contains a readonly directory
 884         d.addCallback(lambda res:
 885                       self.GET(self.public_url, followRedirect=True))
 886         def _check3(res):
 887             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 888                                       '</td>\s+<td>DIR-RO</td>', res))
 889         d.addCallback(_check3)
 890
 891         return d
 892
 893     def test_GET_DIRURL_badtype(self):
 894         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 895                                   400, "Bad Request",
 896                                   "bad t=bogus",
 897                                   self.GET,
 898                                   self.public_url + "/foo?t=bogus")
 899         return d
 900
 901     def test_GET_DIRURL_json(self):
 902         d = self.GET(self.public_url + "/foo?t=json")
 903         d.addCallback(self.failUnlessIsFooJSON)
 904         return d
 905
 906
 907     def test_POST_DIRURL_manifest_no_ophandle(self):
 908         d = self.shouldFail2(error.Error,
 909                              "test_POST_DIRURL_manifest_no_ophandle",
 910                              "400 Bad Request",
 911                              "slow operation requires ophandle=",
 912                              self.POST, self.public_url, t="start-manifest")
 913         return d
 914
 915     def test_POST_DIRURL_manifest(self):
 916         d = defer.succeed(None)
 917         def getman(ignored, output):
 918             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 919                           followRedirect=True)
 920             d.addCallback(self.wait_for_operation, "125")
 921             d.addCallback(self.get_operation_results, "125", output)
 922             return d
 923         d.addCallback(getman, None)
 924         def _got_html(manifest):
 925             self.failUnless("Manifest of SI=" in manifest)
 926             self.failUnless("<td>sub</td>" in manifest)
 927             self.failUnless(self._sub_uri in manifest)
 928             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 929         d.addCallback(_got_html)
 930
 931         # both t=status and unadorned GET should be identical
 932         d.addCallback(lambda res: self.GET("/operations/125"))
 933         d.addCallback(_got_html)
 934
 935         d.addCallback(getman, "html")
 936         d.addCallback(_got_html)
 937         d.addCallback(getman, "text")
 938         def _got_text(manifest):
 939             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 940             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 941         d.addCallback(_got_text)
 942         d.addCallback(getman, "JSON")
 943         def _got_json(res):
 944             data = res["manifest"]
 945             got = {}
 946             for (path_list, cap) in data:
 947                 got[tuple(path_list)] = cap
 948             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 949             self.failUnless((u"sub",u"baz.txt") in got)
 950             self.failUnless("finished" in res)
 951             self.failUnless("origin" in res)
 952             self.failUnless("storage-index" in res)
 953             self.failUnless("verifycaps" in res)
 954             self.failUnless("stats" in res)
 955         d.addCallback(_got_json)
 956         return d
 957
 958     def test_POST_DIRURL_deepsize_no_ophandle(self):
 959         d = self.shouldFail2(error.Error,
 960                              "test_POST_DIRURL_deepsize_no_ophandle",
 961                              "400 Bad Request",
 962                              "slow operation requires ophandle=",
 963                              self.POST, self.public_url, t="start-deep-size")
 964         return d
 965
 966     def test_POST_DIRURL_deepsize(self):
 967         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 968                       followRedirect=True)
 969         d.addCallback(self.wait_for_operation, "126")
 970         d.addCallback(self.get_operation_results, "126", "json")
 971         def _got_json(data):
 972             self.failUnlessEqual(data["finished"], True)
 973             size = data["size"]
 974             self.failUnless(size > 1000)
 975         d.addCallback(_got_json)
 976         d.addCallback(self.get_operation_results, "126", "text")
 977         def _got_text(res):
 978             mo = re.search(r'^size: (\d+)$', res, re.M)
 979             self.failUnless(mo, res)
 980             size = int(mo.group(1))
 981             # with directories, the size varies.
 982             self.failUnless(size > 1000)
 983         d.addCallback(_got_text)
 984         return d
 985
 986     def test_POST_DIRURL_deepstats_no_ophandle(self):
 987         d = self.shouldFail2(error.Error,
 988                              "test_POST_DIRURL_deepstats_no_ophandle",
 989                              "400 Bad Request",
 990                              "slow operation requires ophandle=",
 991                              self.POST, self.public_url, t="start-deep-stats")
 992         return d
 993
 994     def test_POST_DIRURL_deepstats(self):
 995         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 996                       followRedirect=True)
 997         d.addCallback(self.wait_for_operation, "127")
 998         d.addCallback(self.get_operation_results, "127", "json")
 999         def _got_json(stats):
1000             expected = {"count-immutable-files": 3,
1001                         "count-mutable-files": 0,
1002                         "count-literal-files": 0,
1003                         "count-files": 3,
1004                         "count-directories": 3,
1005                         "size-immutable-files": 57,
1006                         "size-literal-files": 0,
1007                         #"size-directories": 1912, # varies
1008                         #"largest-directory": 1590,
1009                         "largest-directory-children": 5,
1010                         "largest-immutable-file": 19,
1011                         }
1012             for k,v in expected.iteritems():
1013                 self.failUnlessEqual(stats[k], v,
1014                                      "stats[%s] was %s, not %s" %
1015                                      (k, stats[k], v))
1016             self.failUnlessEqual(stats["size-files-histogram"],
1017                                  [ [11, 31, 3] ])
1018         d.addCallback(_got_json)
1019         return d
1020
1021     def test_GET_DIRURL_uri(self):
1022         d = self.GET(self.public_url + "/foo?t=uri")
1023         def _check(res):
1024             self.failUnlessEqual(res, self._foo_uri)
1025         d.addCallback(_check)
1026         return d
1027
1028     def test_GET_DIRURL_readonly_uri(self):
1029         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1030         def _check(res):
1031             self.failUnlessEqual(res, self._foo_readonly_uri)
1032         d.addCallback(_check)
1033         return d
1034
1035     def test_PUT_NEWDIRURL(self):
1036         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1037         d.addCallback(lambda res:
1038                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1039         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1040         d.addCallback(self.failUnlessNodeKeysAre, [])
1041         return d
1042
1043     def test_PUT_NEWDIRURL_exists(self):
1044         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1045         d.addCallback(lambda res:
1046                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1047         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1048         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1049         return d
1050
1051     def test_PUT_NEWDIRURL_blocked(self):
1052         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1053                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1054                              self.PUT,
1055                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1056         d.addCallback(lambda res:
1057                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1058         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1059         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1060         return d
1061
1062     def test_PUT_NEWDIRURL_mkdir_p(self):
1063         d = defer.succeed(None)
1064         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1065         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1066         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1067         def mkdir_p(mkpnode):
1068             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1069             d = self.POST(url)
1070             def made_subsub(ssuri):
1071                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1072                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1073                 d = self.POST(url)
1074                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1075                 return d
1076             d.addCallback(made_subsub)
1077             return d
1078         d.addCallback(mkdir_p)
1079         return d
1080
1081     def test_PUT_NEWDIRURL_mkdirs(self):
1082         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1083         d.addCallback(lambda res:
1084                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1085         d.addCallback(lambda res:
1086                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1087         d.addCallback(lambda res:
1088                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1089         d.addCallback(self.failUnlessNodeKeysAre, [])
1090         return d
1091
1092     def test_DELETE_DIRURL(self):
1093         d = self.DELETE(self.public_url + "/foo")
1094         d.addCallback(lambda res:
1095                       self.failIfNodeHasChild(self.public_root, u"foo"))
1096         return d
1097
1098     def test_DELETE_DIRURL_missing(self):
1099         d = self.DELETE(self.public_url + "/foo/missing")
1100         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1101         d.addCallback(lambda res:
1102                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1103         return d
1104
1105     def test_DELETE_DIRURL_missing2(self):
1106         d = self.DELETE(self.public_url + "/missing")
1107         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1108         return d
1109
1110     def dump_root(self):
1111         print "NODEWALK"
1112         w = webish.DirnodeWalkerMixin()
1113         def visitor(childpath, childnode, metadata):
1114             print childpath
1115         d = w.walk(self.public_root, visitor)
1116         return d
1117
1118     def failUnlessNodeKeysAre(self, node, expected_keys):
1119         for k in expected_keys:
1120             assert isinstance(k, unicode)
1121         d = node.list()
1122         def _check(children):
1123             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1124         d.addCallback(_check)
1125         return d
1126     def failUnlessNodeHasChild(self, node, name):
1127         assert isinstance(name, unicode)
1128         d = node.list()
1129         def _check(children):
1130             self.failUnless(name in children)
1131         d.addCallback(_check)
1132         return d
1133     def failIfNodeHasChild(self, node, name):
1134         assert isinstance(name, unicode)
1135         d = node.list()
1136         def _check(children):
1137             self.failIf(name in children)
1138         d.addCallback(_check)
1139         return d
1140
1141     def failUnlessChildContentsAre(self, node, name, expected_contents):
1142         assert isinstance(name, unicode)
1143         d = node.get_child_at_path(name)
1144         d.addCallback(lambda node: node.download_to_data())
1145         def _check(contents):
1146             self.failUnlessEqual(contents, expected_contents)
1147         d.addCallback(_check)
1148         return d
1149
1150     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1151         assert isinstance(name, unicode)
1152         d = node.get_child_at_path(name)
1153         d.addCallback(lambda node: node.download_best_version())
1154         def _check(contents):
1155             self.failUnlessEqual(contents, expected_contents)
1156         d.addCallback(_check)
1157         return d
1158
1159     def failUnlessChildURIIs(self, node, name, expected_uri):
1160         assert isinstance(name, unicode)
1161         d = node.get_child_at_path(name)
1162         def _check(child):
1163             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1164         d.addCallback(_check)
1165         return d
1166
1167     def failUnlessURIMatchesChild(self, got_uri, node, name):
1168         assert isinstance(name, unicode)
1169         d = node.get_child_at_path(name)
1170         def _check(child):
1171             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1172         d.addCallback(_check)
1173         return d
1174
1175     def failUnlessCHKURIHasContents(self, got_uri, contents):
1176         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1177
1178     def test_POST_upload(self):
1179         d = self.POST(self.public_url + "/foo", t="upload",
1180                       file=("new.txt", self.NEWFILE_CONTENTS))
1181         fn = self._foo_node
1182         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1183         d.addCallback(lambda res:
1184                       self.failUnlessChildContentsAre(fn, u"new.txt",
1185                                                       self.NEWFILE_CONTENTS))
1186         return d
1187
1188     def test_POST_upload_unicode(self):
1189         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1190         d = self.POST(self.public_url + "/foo", t="upload",
1191                       file=(filename, self.NEWFILE_CONTENTS))
1192         fn = self._foo_node
1193         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1194         d.addCallback(lambda res:
1195                       self.failUnlessChildContentsAre(fn, filename,
1196                                                       self.NEWFILE_CONTENTS))
1197         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1198         d.addCallback(lambda res: self.GET(target_url))
1199         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1200                                                             self.NEWFILE_CONTENTS,
1201                                                             contents))
1202         return d
1203
1204     def test_POST_upload_unicode_named(self):
1205         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1206         d = self.POST(self.public_url + "/foo", t="upload",
1207                       name=filename,
1208                       file=("overridden", self.NEWFILE_CONTENTS))
1209         fn = self._foo_node
1210         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1211         d.addCallback(lambda res:
1212                       self.failUnlessChildContentsAre(fn, filename,
1213                                                       self.NEWFILE_CONTENTS))
1214         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1215         d.addCallback(lambda res: self.GET(target_url))
1216         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1217                                                             self.NEWFILE_CONTENTS,
1218                                                             contents))
1219         return d
1220
1221     def test_POST_upload_no_link(self):
1222         d = self.POST("/uri", t="upload",
1223                       file=("new.txt", self.NEWFILE_CONTENTS))
1224         def _check_upload_results(page):
1225             # this should be a page which describes the results of the upload
1226             # that just finished.
1227             self.failUnless("Upload Results:" in page)
1228             self.failUnless("URI:" in page)
1229             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1230             mo = uri_re.search(page)
1231             self.failUnless(mo, page)
1232             new_uri = mo.group(1)
1233             return new_uri
1234         d.addCallback(_check_upload_results)
1235         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1236         return d
1237
1238     def test_POST_upload_no_link_whendone(self):
1239         d = self.POST("/uri", t="upload", when_done="/",
1240                       file=("new.txt", self.NEWFILE_CONTENTS))
1241         d.addBoth(self.shouldRedirect, "/")
1242         return d
1243
1244     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1245         d = defer.maybeDeferred(callable, *args, **kwargs)
1246         def done(res):
1247             if isinstance(res, failure.Failure):
1248                 res.trap(error.PageRedirect)
1249                 statuscode = res.value.status
1250                 target = res.value.location
1251                 return checker(statuscode, target)
1252             self.fail("%s: callable was supposed to redirect, not return '%s'"
1253                       % (which, res))
1254         d.addBoth(done)
1255         return d
1256
1257     def test_POST_upload_no_link_whendone_results(self):
1258         def check(statuscode, target):
1259             self.failUnlessEqual(statuscode, str(http.FOUND))
1260             self.failUnless(target.startswith(self.webish_url), target)
1261             return client.getPage(target, method="GET")
1262         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1263                                  check,
1264                                  self.POST, "/uri", t="upload",
1265                                  when_done="/uri/%(uri)s",
1266                                  file=("new.txt", self.NEWFILE_CONTENTS))
1267         d.addCallback(lambda res:
1268                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1269         return d
1270
1271     def test_POST_upload_no_link_mutable(self):
1272         d = self.POST("/uri", t="upload", mutable="true",
1273                       file=("new.txt", self.NEWFILE_CONTENTS))
1274         def _check(new_uri):
1275             new_uri = new_uri.strip()
1276             self.new_uri = new_uri
1277             u = IURI(new_uri)
1278             self.failUnless(IMutableFileURI.providedBy(u))
1279             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1280             n = self.s.create_node_from_uri(new_uri)
1281             return n.download_best_version()
1282         d.addCallback(_check)
1283         def _check2(data):
1284             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1285             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1286         d.addCallback(_check2)
1287         def _check3(data):
1288             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1289             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1290         d.addCallback(_check3)
1291         def _check4(data):
1292             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1293         d.addCallback(_check4)
1294         return d
1295
1296     def test_POST_upload_no_link_mutable_toobig(self):
1297         d = self.shouldFail2(error.Error,
1298                              "test_POST_upload_no_link_mutable_toobig",
1299                              "413 Request Entity Too Large",
1300                              "SDMF is limited to one segment, and 10001 > 10000",
1301                              self.POST,
1302                              "/uri", t="upload", mutable="true",
1303                              file=("new.txt",
1304                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1305         return d
1306
1307     def test_POST_upload_mutable(self):
1308         # this creates a mutable file
1309         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1310                       file=("new.txt", self.NEWFILE_CONTENTS))
1311         fn = self._foo_node
1312         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1313         d.addCallback(lambda res:
1314                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1315                                                              self.NEWFILE_CONTENTS))
1316         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1317         def _got(newnode):
1318             self.failUnless(IMutableFileNode.providedBy(newnode))
1319             self.failUnless(newnode.is_mutable())
1320             self.failIf(newnode.is_readonly())
1321             self._mutable_node = newnode
1322             self._mutable_uri = newnode.get_uri()
1323         d.addCallback(_got)
1324
1325         # now upload it again and make sure that the URI doesn't change
1326         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1327         d.addCallback(lambda res:
1328                       self.POST(self.public_url + "/foo", t="upload",
1329                                 mutable="true",
1330                                 file=("new.txt", NEWER_CONTENTS)))
1331         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1332         d.addCallback(lambda res:
1333                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1334                                                              NEWER_CONTENTS))
1335         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1336         def _got2(newnode):
1337             self.failUnless(IMutableFileNode.providedBy(newnode))
1338             self.failUnless(newnode.is_mutable())
1339             self.failIf(newnode.is_readonly())
1340             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1341         d.addCallback(_got2)
1342
1343         # upload a second time, using PUT instead of POST
1344         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1345         d.addCallback(lambda res:
1346                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1347         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1348         d.addCallback(lambda res:
1349                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1350                                                              NEW2_CONTENTS))
1351
1352         # finally list the directory, since mutable files are displayed
1353         # slightly differently
1354
1355         d.addCallback(lambda res:
1356                       self.GET(self.public_url + "/foo/",
1357                                followRedirect=True))
1358         def _check_page(res):
1359             # TODO: assert more about the contents
1360             self.failUnless("SSK" in res)
1361             return res
1362         d.addCallback(_check_page)
1363
1364         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1365         def _got3(newnode):
1366             self.failUnless(IMutableFileNode.providedBy(newnode))
1367             self.failUnless(newnode.is_mutable())
1368             self.failIf(newnode.is_readonly())
1369             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1370         d.addCallback(_got3)
1371
1372         # look at the JSON form of the enclosing directory
1373         d.addCallback(lambda res:
1374                       self.GET(self.public_url + "/foo/?t=json",
1375                                followRedirect=True))
1376         def _check_page_json(res):
1377             parsed = simplejson.loads(res)
1378             self.failUnlessEqual(parsed[0], "dirnode")
1379             children = dict( [(unicode(name),value)
1380                               for (name,value)
1381                               in parsed[1]["children"].iteritems()] )
1382             self.failUnless("new.txt" in children)
1383             new_json = children["new.txt"]
1384             self.failUnlessEqual(new_json[0], "filenode")
1385             self.failUnless(new_json[1]["mutable"])
1386             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1387             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1388             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1389         d.addCallback(_check_page_json)
1390
1391         # and the JSON form of the file
1392         d.addCallback(lambda res:
1393                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1394         def _check_file_json(res):
1395             parsed = simplejson.loads(res)
1396             self.failUnlessEqual(parsed[0], "filenode")
1397             self.failUnless(parsed[1]["mutable"])
1398             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1399             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1400             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1401         d.addCallback(_check_file_json)
1402
1403         # and look at t=uri and t=readonly-uri
1404         d.addCallback(lambda res:
1405                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1406         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1409         def _check_ro_uri(res):
1410             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1411             self.failUnlessEqual(res, ro_uri)
1412         d.addCallback(_check_ro_uri)
1413
1414         # make sure we can get to it from /uri/URI
1415         d.addCallback(lambda res:
1416                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1417         d.addCallback(lambda res:
1418                       self.failUnlessEqual(res, NEW2_CONTENTS))
1419
1420         # and that HEAD computes the size correctly
1421         d.addCallback(lambda res:
1422                       self.HEAD(self.public_url + "/foo/new.txt",
1423                                 return_response=True))
1424         def _got_headers((res, status, headers)):
1425             self.failUnlessEqual(res, "")
1426             self.failUnlessEqual(headers["content-length"][0],
1427                                  str(len(NEW2_CONTENTS)))
1428             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1429         d.addCallback(_got_headers)
1430
1431         # make sure that size errors are displayed correctly for overwrite
1432         d.addCallback(lambda res:
1433                       self.shouldFail2(error.Error,
1434                                        "test_POST_upload_mutable-toobig",
1435                                        "413 Request Entity Too Large",
1436                                        "SDMF is limited to one segment, and 10001 > 10000",
1437                                        self.POST,
1438                                        self.public_url + "/foo", t="upload",
1439                                        mutable="true",
1440                                        file=("new.txt",
1441                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1442                                        ))
1443
1444         d.addErrback(self.dump_error)
1445         return d
1446
1447     def test_POST_upload_mutable_toobig(self):
1448         d = self.shouldFail2(error.Error,
1449                              "test_POST_upload_no_link_mutable_toobig",
1450                              "413 Request Entity Too Large",
1451                              "SDMF is limited to one segment, and 10001 > 10000",
1452                              self.POST,
1453                              self.public_url + "/foo",
1454                              t="upload", mutable="true",
1455                              file=("new.txt",
1456                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1457         return d
1458
1459     def dump_error(self, f):
1460         # if the web server returns an error code (like 400 Bad Request),
1461         # web.client.getPage puts the HTTP response body into the .response
1462         # attribute of the exception object that it gives back. It does not
1463         # appear in the Failure's repr(), so the ERROR that trial displays
1464         # will be rather terse and unhelpful. addErrback this method to the
1465         # end of your chain to get more information out of these errors.
1466         if f.check(error.Error):
1467             print "web.error.Error:"
1468             print f
1469             print f.value.response
1470         return f
1471
1472     def test_POST_upload_replace(self):
1473         d = self.POST(self.public_url + "/foo", t="upload",
1474                       file=("bar.txt", self.NEWFILE_CONTENTS))
1475         fn = self._foo_node
1476         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1477         d.addCallback(lambda res:
1478                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1479                                                       self.NEWFILE_CONTENTS))
1480         return d
1481
1482     def test_POST_upload_no_replace_ok(self):
1483         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1484                       file=("new.txt", self.NEWFILE_CONTENTS))
1485         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1486         d.addCallback(lambda res: self.failUnlessEqual(res,
1487                                                        self.NEWFILE_CONTENTS))
1488         return d
1489
1490     def test_POST_upload_no_replace_queryarg(self):
1491         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1492                       file=("bar.txt", self.NEWFILE_CONTENTS))
1493         d.addBoth(self.shouldFail, error.Error,
1494                   "POST_upload_no_replace_queryarg",
1495                   "409 Conflict",
1496                   "There was already a child by that name, and you asked me "
1497                   "to not replace it")
1498         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1499         d.addCallback(self.failUnlessIsBarDotTxt)
1500         return d
1501
1502     def test_POST_upload_no_replace_field(self):
1503         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1504                       file=("bar.txt", self.NEWFILE_CONTENTS))
1505         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1506                   "409 Conflict",
1507                   "There was already a child by that name, and you asked me "
1508                   "to not replace it")
1509         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1510         d.addCallback(self.failUnlessIsBarDotTxt)
1511         return d
1512
1513     def test_POST_upload_whendone(self):
1514         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1515                       file=("new.txt", self.NEWFILE_CONTENTS))
1516         d.addBoth(self.shouldRedirect, "/THERE")
1517         fn = self._foo_node
1518         d.addCallback(lambda res:
1519                       self.failUnlessChildContentsAre(fn, u"new.txt",
1520                                                       self.NEWFILE_CONTENTS))
1521         return d
1522
1523     def test_POST_upload_named(self):
1524         fn = self._foo_node
1525         d = self.POST(self.public_url + "/foo", t="upload",
1526                       name="new.txt", file=self.NEWFILE_CONTENTS)
1527         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1528         d.addCallback(lambda res:
1529                       self.failUnlessChildContentsAre(fn, u"new.txt",
1530                                                       self.NEWFILE_CONTENTS))
1531         return d
1532
1533     def test_POST_upload_named_badfilename(self):
1534         d = self.POST(self.public_url + "/foo", t="upload",
1535                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1536         d.addBoth(self.shouldFail, error.Error,
1537                   "test_POST_upload_named_badfilename",
1538                   "400 Bad Request",
1539                   "name= may not contain a slash",
1540                   )
1541         # make sure that nothing was added
1542         d.addCallback(lambda res:
1543                       self.failUnlessNodeKeysAre(self._foo_node,
1544                                                  [u"bar.txt", u"blockingfile",
1545                                                   u"empty", u"n\u00fc.txt",
1546                                                   u"sub"]))
1547         return d
1548
1549     def test_POST_FILEURL_check(self):
1550         bar_url = self.public_url + "/foo/bar.txt"
1551         d = self.POST(bar_url, t="check")
1552         def _check(res):
1553             self.failUnless("Healthy :" in res)
1554         d.addCallback(_check)
1555         redir_url = "http://allmydata.org/TARGET"
1556         def _check2(statuscode, target):
1557             self.failUnlessEqual(statuscode, str(http.FOUND))
1558             self.failUnlessEqual(target, redir_url)
1559         d.addCallback(lambda res:
1560                       self.shouldRedirect2("test_POST_FILEURL_check",
1561                                            _check2,
1562                                            self.POST, bar_url,
1563                                            t="check",
1564                                            when_done=redir_url))
1565         d.addCallback(lambda res:
1566                       self.POST(bar_url, t="check", return_to=redir_url))
1567         def _check3(res):
1568             self.failUnless("Healthy :" in res)
1569             self.failUnless("Return to parent directory" in res)
1570             self.failUnless(redir_url in res)
1571         d.addCallback(_check3)
1572
1573         d.addCallback(lambda res:
1574                       self.POST(bar_url, t="check", output="JSON"))
1575         def _check_json(res):
1576             data = simplejson.loads(res)
1577             self.failUnless("storage-index" in data)
1578             self.failUnless(data["results"]["healthy"])
1579         d.addCallback(_check_json)
1580
1581         return d
1582
1583     def test_POST_FILEURL_check_and_repair(self):
1584         bar_url = self.public_url + "/foo/bar.txt"
1585         d = self.POST(bar_url, t="check", repair="true")
1586         def _check(res):
1587             self.failUnless("Healthy :" in res)
1588         d.addCallback(_check)
1589         redir_url = "http://allmydata.org/TARGET"
1590         def _check2(statuscode, target):
1591             self.failUnlessEqual(statuscode, str(http.FOUND))
1592             self.failUnlessEqual(target, redir_url)
1593         d.addCallback(lambda res:
1594                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1595                                            _check2,
1596                                            self.POST, bar_url,
1597                                            t="check", repair="true",
1598                                            when_done=redir_url))
1599         d.addCallback(lambda res:
1600                       self.POST(bar_url, t="check", return_to=redir_url))
1601         def _check3(res):
1602             self.failUnless("Healthy :" in res)
1603             self.failUnless("Return to parent directory" in res)
1604             self.failUnless(redir_url in res)
1605         d.addCallback(_check3)
1606         return d
1607
1608     def test_POST_DIRURL_check(self):
1609         foo_url = self.public_url + "/foo/"
1610         d = self.POST(foo_url, t="check")
1611         def _check(res):
1612             self.failUnless("Healthy :" in res, res)
1613         d.addCallback(_check)
1614         redir_url = "http://allmydata.org/TARGET"
1615         def _check2(statuscode, target):
1616             self.failUnlessEqual(statuscode, str(http.FOUND))
1617             self.failUnlessEqual(target, redir_url)
1618         d.addCallback(lambda res:
1619                       self.shouldRedirect2("test_POST_DIRURL_check",
1620                                            _check2,
1621                                            self.POST, foo_url,
1622                                            t="check",
1623                                            when_done=redir_url))
1624         d.addCallback(lambda res:
1625                       self.POST(foo_url, t="check", return_to=redir_url))
1626         def _check3(res):
1627             self.failUnless("Healthy :" in res, res)
1628             self.failUnless("Return to parent directory" in res)
1629             self.failUnless(redir_url in res)
1630         d.addCallback(_check3)
1631
1632         d.addCallback(lambda res:
1633                       self.POST(foo_url, t="check", output="JSON"))
1634         def _check_json(res):
1635             data = simplejson.loads(res)
1636             self.failUnless("storage-index" in data)
1637             self.failUnless(data["results"]["healthy"])
1638         d.addCallback(_check_json)
1639
1640         return d
1641
1642     def test_POST_DIRURL_check_and_repair(self):
1643         foo_url = self.public_url + "/foo/"
1644         d = self.POST(foo_url, t="check", repair="true")
1645         def _check(res):
1646             self.failUnless("Healthy :" in res, res)
1647         d.addCallback(_check)
1648         redir_url = "http://allmydata.org/TARGET"
1649         def _check2(statuscode, target):
1650             self.failUnlessEqual(statuscode, str(http.FOUND))
1651             self.failUnlessEqual(target, redir_url)
1652         d.addCallback(lambda res:
1653                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1654                                            _check2,
1655                                            self.POST, foo_url,
1656                                            t="check", repair="true",
1657                                            when_done=redir_url))
1658         d.addCallback(lambda res:
1659                       self.POST(foo_url, t="check", return_to=redir_url))
1660         def _check3(res):
1661             self.failUnless("Healthy :" in res)
1662             self.failUnless("Return to parent directory" in res)
1663             self.failUnless(redir_url in res)
1664         d.addCallback(_check3)
1665         return d
1666
1667     def wait_for_operation(self, ignored, ophandle):
1668         url = "/operations/" + ophandle
1669         url += "?t=status&output=JSON"
1670         d = self.GET(url)
1671         def _got(res):
1672             data = simplejson.loads(res)
1673             if not data["finished"]:
1674                 d = self.stall(delay=1.0)
1675                 d.addCallback(self.wait_for_operation, ophandle)
1676                 return d
1677             return data
1678         d.addCallback(_got)
1679         return d
1680
1681     def get_operation_results(self, ignored, ophandle, output=None):
1682         url = "/operations/" + ophandle
1683         url += "?t=status"
1684         if output:
1685             url += "&output=" + output
1686         d = self.GET(url)
1687         def _got(res):
1688             if output and output.lower() == "json":
1689                 return simplejson.loads(res)
1690             return res
1691         d.addCallback(_got)
1692         return d
1693
1694     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1695         d = self.shouldFail2(error.Error,
1696                              "test_POST_DIRURL_deepcheck_no_ophandle",
1697                              "400 Bad Request",
1698                              "slow operation requires ophandle=",
1699                              self.POST, self.public_url, t="start-deep-check")
1700         return d
1701
1702     def test_POST_DIRURL_deepcheck(self):
1703         def _check_redirect(statuscode, target):
1704             self.failUnlessEqual(statuscode, str(http.FOUND))
1705             self.failUnless(target.endswith("/operations/123"))
1706         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1707                                  self.POST, self.public_url,
1708                                  t="start-deep-check", ophandle="123")
1709         d.addCallback(self.wait_for_operation, "123")
1710         def _check_json(data):
1711             self.failUnlessEqual(data["finished"], True)
1712             self.failUnlessEqual(data["count-objects-checked"], 8)
1713             self.failUnlessEqual(data["count-objects-healthy"], 8)
1714         d.addCallback(_check_json)
1715         d.addCallback(self.get_operation_results, "123", "html")
1716         def _check_html(res):
1717             self.failUnless("Objects Checked: <span>8</span>" in res)
1718             self.failUnless("Objects Healthy: <span>8</span>" in res)
1719         d.addCallback(_check_html)
1720
1721         d.addCallback(lambda res:
1722                       self.GET("/operations/123/"))
1723         d.addCallback(_check_html) # should be the same as without the slash
1724
1725         d.addCallback(lambda res:
1726                       self.shouldFail2(error.Error, "one", "404 Not Found",
1727                                        "No detailed results for SI bogus",
1728                                        self.GET, "/operations/123/bogus"))
1729
1730         foo_si = self._foo_node.get_storage_index()
1731         foo_si_s = base32.b2a(foo_si)
1732         d.addCallback(lambda res:
1733                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1734         def _check_foo_json(res):
1735             data = simplejson.loads(res)
1736             self.failUnlessEqual(data["storage-index"], foo_si_s)
1737             self.failUnless(data["results"]["healthy"])
1738         d.addCallback(_check_foo_json)
1739         return d
1740
1741     def test_POST_DIRURL_deepcheck_and_repair(self):
1742         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1743                       ophandle="124", output="json", followRedirect=True)
1744         d.addCallback(self.wait_for_operation, "124")
1745         def _check_json(data):
1746             self.failUnlessEqual(data["finished"], True)
1747             self.failUnlessEqual(data["count-objects-checked"], 8)
1748             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1749             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1750             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1751             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1752             self.failUnlessEqual(data["count-repairs-successful"], 0)
1753             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1754             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1755             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1756             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1757         d.addCallback(_check_json)
1758         d.addCallback(self.get_operation_results, "124", "html")
1759         def _check_html(res):
1760             self.failUnless("Objects Checked: <span>8</span>" in res)
1761
1762             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1763             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1764             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1765
1766             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1767             self.failUnless("Repairs Successful: <span>0</span>" in res)
1768             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1769
1770             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1771             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1772             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1773         d.addCallback(_check_html)
1774         return d
1775
1776     def test_POST_FILEURL_bad_t(self):
1777         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1778                              "POST to file: bad t=bogus",
1779                              self.POST, self.public_url + "/foo/bar.txt",
1780                              t="bogus")
1781         return d
1782
1783     def test_POST_mkdir(self): # return value?
1784         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1785         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1786         d.addCallback(self.failUnlessNodeKeysAre, [])
1787         return d
1788
1789     def test_POST_mkdir_2(self):
1790         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1791         d.addCallback(lambda res:
1792                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794         d.addCallback(self.failUnlessNodeKeysAre, [])
1795         return d
1796
1797     def test_POST_mkdirs_2(self):
1798         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1799         d.addCallback(lambda res:
1800                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1801         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1802         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1803         d.addCallback(self.failUnlessNodeKeysAre, [])
1804         return d
1805
1806     def test_POST_mkdir_no_parentdir_noredirect(self):
1807         d = self.POST("/uri?t=mkdir")
1808         def _after_mkdir(res):
1809             uri.NewDirectoryURI.init_from_string(res)
1810         d.addCallback(_after_mkdir)
1811         return d
1812
1813     def test_POST_mkdir_no_parentdir_redirect(self):
1814         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1815         d.addBoth(self.shouldRedirect, None, statuscode='303')
1816         def _check_target(target):
1817             target = urllib.unquote(target)
1818             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1819         d.addCallback(_check_target)
1820         return d
1821
1822     def test_POST_noparent_bad(self):
1823         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1824                                   "/uri accepts only PUT, PUT?t=mkdir, "
1825                                   "POST?t=upload, and POST?t=mkdir",
1826                                   self.POST, "/uri?t=bogus")
1827         return d
1828
1829     def test_welcome_page_mkdir_button(self):
1830         # Fetch the welcome page.
1831         d = self.GET("/")
1832         def _after_get_welcome_page(res):
1833             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1834             mo = MKDIR_BUTTON_RE.search(res)
1835             formaction = mo.group(1)
1836             formt = mo.group(2)
1837             formaname = mo.group(3)
1838             formavalue = mo.group(4)
1839             return (formaction, formt, formaname, formavalue)
1840         d.addCallback(_after_get_welcome_page)
1841         def _after_parse_form(res):
1842             (formaction, formt, formaname, formavalue) = res
1843             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1844         d.addCallback(_after_parse_form)
1845         d.addBoth(self.shouldRedirect, None, statuscode='303')
1846         return d
1847
1848     def test_POST_mkdir_replace(self): # return value?
1849         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1850         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1851         d.addCallback(self.failUnlessNodeKeysAre, [])
1852         return d
1853
1854     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1855         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1856         d.addBoth(self.shouldFail, error.Error,
1857                   "POST_mkdir_no_replace_queryarg",
1858                   "409 Conflict",
1859                   "There was already a child by that name, and you asked me "
1860                   "to not replace it")
1861         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1862         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1863         return d
1864
1865     def test_POST_mkdir_no_replace_field(self): # return value?
1866         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1867                       replace="false")
1868         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1869                   "409 Conflict",
1870                   "There was already a child by that name, and you asked me "
1871                   "to not replace it")
1872         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1873         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1874         return d
1875
1876     def test_POST_mkdir_whendone_field(self):
1877         d = self.POST(self.public_url + "/foo",
1878                       t="mkdir", name="newdir", when_done="/THERE")
1879         d.addBoth(self.shouldRedirect, "/THERE")
1880         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1881         d.addCallback(self.failUnlessNodeKeysAre, [])
1882         return d
1883
1884     def test_POST_mkdir_whendone_queryarg(self):
1885         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1886                       t="mkdir", name="newdir")
1887         d.addBoth(self.shouldRedirect, "/THERE")
1888         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1889         d.addCallback(self.failUnlessNodeKeysAre, [])
1890         return d
1891
1892     def test_POST_bad_t(self):
1893         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1894                              "POST to a directory with bad t=BOGUS",
1895                              self.POST, self.public_url + "/foo", t="BOGUS")
1896         return d
1897
1898     def test_POST_set_children(self):
1899         contents9, n9, newuri9 = self.makefile(9)
1900         contents10, n10, newuri10 = self.makefile(10)
1901         contents11, n11, newuri11 = self.makefile(11)
1902
1903         reqbody = """{
1904                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1905                                                 "size": 0,
1906                                                 "metadata": {
1907                                                   "ctime": 1002777696.7564139,
1908                                                   "mtime": 1002777696.7564139
1909                                                  }
1910                                                } ],
1911                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1912                                                 "size": 1,
1913                                                 "metadata": {
1914                                                   "ctime": 1002777696.7564139,
1915                                                   "mtime": 1002777696.7564139
1916                                                  }
1917                                                } ],
1918                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1919                                                 "size": 2,
1920                                                 "metadata": {
1921                                                   "ctime": 1002777696.7564139,
1922                                                   "mtime": 1002777696.7564139
1923                                                  }
1924                                                } ]
1925                     }""" % (newuri9, newuri10, newuri11)
1926
1927         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1928
1929         d = client.getPage(url, method="POST", postdata=reqbody)
1930         def _then(res):
1931             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1932             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1933             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1934
1935         d.addCallback(_then)
1936         d.addErrback(self.dump_error)
1937         return d
1938
1939     def test_POST_put_uri(self):
1940         contents, n, newuri = self.makefile(8)
1941         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1942         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1943         d.addCallback(lambda res:
1944                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1945                                                       contents))
1946         return d
1947
1948     def test_POST_put_uri_replace(self):
1949         contents, n, newuri = self.makefile(8)
1950         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1951         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1952         d.addCallback(lambda res:
1953                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1954                                                       contents))
1955         return d
1956
1957     def test_POST_put_uri_no_replace_queryarg(self):
1958         contents, n, newuri = self.makefile(8)
1959         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1960                       name="bar.txt", uri=newuri)
1961         d.addBoth(self.shouldFail, error.Error,
1962                   "POST_put_uri_no_replace_queryarg",
1963                   "409 Conflict",
1964                   "There was already a child by that name, and you asked me "
1965                   "to not replace it")
1966         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1967         d.addCallback(self.failUnlessIsBarDotTxt)
1968         return d
1969
1970     def test_POST_put_uri_no_replace_field(self):
1971         contents, n, newuri = self.makefile(8)
1972         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1973                       name="bar.txt", uri=newuri)
1974         d.addBoth(self.shouldFail, error.Error,
1975                   "POST_put_uri_no_replace_field",
1976                   "409 Conflict",
1977                   "There was already a child by that name, and you asked me "
1978                   "to not replace it")
1979         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1980         d.addCallback(self.failUnlessIsBarDotTxt)
1981         return d
1982
1983     def test_POST_delete(self):
1984         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1985         d.addCallback(lambda res: self._foo_node.list())
1986         def _check(children):
1987             self.failIf(u"bar.txt" in children)
1988         d.addCallback(_check)
1989         return d
1990
1991     def test_POST_rename_file(self):
1992         d = self.POST(self.public_url + "/foo", t="rename",
1993                       from_name="bar.txt", to_name='wibble.txt')
1994         d.addCallback(lambda res:
1995                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1996         d.addCallback(lambda res:
1997                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1998         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1999         d.addCallback(self.failUnlessIsBarDotTxt)
2000         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2001         d.addCallback(self.failUnlessIsBarJSON)
2002         return d
2003
2004     def test_POST_rename_file_redundant(self):
2005         d = self.POST(self.public_url + "/foo", t="rename",
2006                       from_name="bar.txt", to_name='bar.txt')
2007         d.addCallback(lambda res:
2008                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2009         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2010         d.addCallback(self.failUnlessIsBarDotTxt)
2011         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2012         d.addCallback(self.failUnlessIsBarJSON)
2013         return d
2014
2015     def test_POST_rename_file_replace(self):
2016         # rename a file and replace a directory with it
2017         d = self.POST(self.public_url + "/foo", t="rename",
2018                       from_name="bar.txt", to_name='empty')
2019         d.addCallback(lambda res:
2020                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2021         d.addCallback(lambda res:
2022                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2023         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2024         d.addCallback(self.failUnlessIsBarDotTxt)
2025         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2026         d.addCallback(self.failUnlessIsBarJSON)
2027         return d
2028
2029     def test_POST_rename_file_no_replace_queryarg(self):
2030         # rename a file and replace a directory with it
2031         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2032                       from_name="bar.txt", to_name='empty')
2033         d.addBoth(self.shouldFail, error.Error,
2034                   "POST_rename_file_no_replace_queryarg",
2035                   "409 Conflict",
2036                   "There was already a child by that name, and you asked me "
2037                   "to not replace it")
2038         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2039         d.addCallback(self.failUnlessIsEmptyJSON)
2040         return d
2041
2042     def test_POST_rename_file_no_replace_field(self):
2043         # rename a file and replace a directory with it
2044         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2045                       from_name="bar.txt", to_name='empty')
2046         d.addBoth(self.shouldFail, error.Error,
2047                   "POST_rename_file_no_replace_field",
2048                   "409 Conflict",
2049                   "There was already a child by that name, and you asked me "
2050                   "to not replace it")
2051         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2052         d.addCallback(self.failUnlessIsEmptyJSON)
2053         return d
2054
2055     def failUnlessIsEmptyJSON(self, res):
2056         data = simplejson.loads(res)
2057         self.failUnlessEqual(data[0], "dirnode", data)
2058         self.failUnlessEqual(len(data[1]["children"]), 0)
2059
2060     def test_POST_rename_file_slash_fail(self):
2061         d = self.POST(self.public_url + "/foo", t="rename",
2062                       from_name="bar.txt", to_name='kirk/spock.txt')
2063         d.addBoth(self.shouldFail, error.Error,
2064                   "test_POST_rename_file_slash_fail",
2065                   "400 Bad Request",
2066                   "to_name= may not contain a slash",
2067                   )
2068         d.addCallback(lambda res:
2069                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2070         return d
2071
2072     def test_POST_rename_dir(self):
2073         d = self.POST(self.public_url, t="rename",
2074                       from_name="foo", to_name='plunk')
2075         d.addCallback(lambda res:
2076                       self.failIfNodeHasChild(self.public_root, u"foo"))
2077         d.addCallback(lambda res:
2078                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2079         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2080         d.addCallback(self.failUnlessIsFooJSON)
2081         return d
2082
2083     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2084         """ If target is not None then the redirection has to go to target.  If
2085         statuscode is not None then the redirection has to be accomplished with
2086         that HTTP status code."""
2087         if not isinstance(res, failure.Failure):
2088             to_where = (target is None) and "somewhere" or ("to " + target)
2089             self.fail("%s: we were expecting to get redirected %s, not get an"
2090                       " actual page: %s" % (which, to_where, res))
2091         res.trap(error.PageRedirect)
2092         if statuscode is not None:
2093             self.failUnlessEqual(res.value.status, statuscode,
2094                                  "%s: not a redirect" % which)
2095         if target is not None:
2096             # the PageRedirect does not seem to capture the uri= query arg
2097             # properly, so we can't check for it.
2098             realtarget = self.webish_url + target
2099             self.failUnlessEqual(res.value.location, realtarget,
2100                                  "%s: wrong target" % which)
2101         return res.value.location
2102
2103     def test_GET_URI_form(self):
2104         base = "/uri?uri=%s" % self._bar_txt_uri
2105         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2106         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2107         d = self.GET(base)
2108         d.addBoth(self.shouldRedirect, targetbase)
2109         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2110         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2111         d.addCallback(lambda res: self.GET(base+"&t=json"))
2112         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2113         d.addCallback(self.log, "about to get file by uri")
2114         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2115         d.addCallback(self.failUnlessIsBarDotTxt)
2116         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2117         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2118                                            followRedirect=True))
2119         d.addCallback(self.failUnlessIsFooJSON)
2120         d.addCallback(self.log, "got dir by uri")
2121
2122         return d
2123
2124     def test_GET_URI_form_bad(self):
2125         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2126                              "400 Bad Request", "GET /uri requires uri=",
2127                              self.GET, "/uri")
2128         return d
2129
2130     def test_GET_rename_form(self):
2131         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2132                      followRedirect=True)
2133         def _check(res):
2134             self.failUnless('name="when_done" value="."' in res, res)
2135             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2136         d.addCallback(_check)
2137         return d
2138
2139     def log(self, res, msg):
2140         #print "MSG: %s  RES: %s" % (msg, res)
2141         log.msg(msg)
2142         return res
2143
2144     def test_GET_URI_URL(self):
2145         base = "/uri/%s" % self._bar_txt_uri
2146         d = self.GET(base)
2147         d.addCallback(self.failUnlessIsBarDotTxt)
2148         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2149         d.addCallback(self.failUnlessIsBarDotTxt)
2150         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2151         d.addCallback(self.failUnlessIsBarDotTxt)
2152         return d
2153
2154     def test_GET_URI_URL_dir(self):
2155         base = "/uri/%s?t=json" % self._foo_uri
2156         d = self.GET(base)
2157         d.addCallback(self.failUnlessIsFooJSON)
2158         return d
2159
2160     def test_GET_URI_URL_missing(self):
2161         base = "/uri/%s" % self._bad_file_uri
2162         d = self.GET(base)
2163         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2164                   http.GONE, response_substring="NotEnoughSharesError")
2165         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2166         # here? we must arrange for a download to fail after target.open()
2167         # has been called, and then inspect the response to see that it is
2168         # shorter than we expected.
2169         return d
2170
2171     def test_PUT_DIRURL_uri(self):
2172         d = self.s.create_empty_dirnode()
2173         def _made_dir(dn):
2174             new_uri = dn.get_uri()
2175             # replace /foo with a new (empty) directory
2176             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2177             d.addCallback(lambda res:
2178                           self.failUnlessEqual(res.strip(), new_uri))
2179             d.addCallback(lambda res:
2180                           self.failUnlessChildURIIs(self.public_root,
2181                                                     u"foo",
2182                                                     new_uri))
2183             return d
2184         d.addCallback(_made_dir)
2185         return d
2186
2187     def test_PUT_DIRURL_uri_noreplace(self):
2188         d = self.s.create_empty_dirnode()
2189         def _made_dir(dn):
2190             new_uri = dn.get_uri()
2191             # replace /foo with a new (empty) directory, but ask that
2192             # replace=false, so it should fail
2193             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2194                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2195                                  self.PUT,
2196                                  self.public_url + "/foo?t=uri&replace=false",
2197                                  new_uri)
2198             d.addCallback(lambda res:
2199                           self.failUnlessChildURIIs(self.public_root,
2200                                                     u"foo",
2201                                                     self._foo_uri))
2202             return d
2203         d.addCallback(_made_dir)
2204         return d
2205
2206     def test_PUT_DIRURL_bad_t(self):
2207         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2208                                  "400 Bad Request", "PUT to a directory",
2209                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2210         d.addCallback(lambda res:
2211                       self.failUnlessChildURIIs(self.public_root,
2212                                                 u"foo",
2213                                                 self._foo_uri))
2214         return d
2215
2216     def test_PUT_NEWFILEURL_uri(self):
2217         contents, n, new_uri = self.makefile(8)
2218         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2219         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2220         d.addCallback(lambda res:
2221                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2222                                                       contents))
2223         return d
2224
2225     def test_PUT_NEWFILEURL_uri_replace(self):
2226         contents, n, new_uri = self.makefile(8)
2227         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2228         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2229         d.addCallback(lambda res:
2230                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2231                                                       contents))
2232         return d
2233
2234     def test_PUT_NEWFILEURL_uri_no_replace(self):
2235         contents, n, new_uri = self.makefile(8)
2236         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2237         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2238                   "409 Conflict",
2239                   "There was already a child by that name, and you asked me "
2240                   "to not replace it")
2241         return d
2242
2243     def test_PUT_NEWFILE_URI(self):
2244         file_contents = "New file contents here\n"
2245         d = self.PUT("/uri", file_contents)
2246         def _check(uri):
2247             self.failUnless(uri in FakeCHKFileNode.all_contents)
2248             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2249                                  file_contents)
2250             return self.GET("/uri/%s" % uri)
2251         d.addCallback(_check)
2252         def _check2(res):
2253             self.failUnlessEqual(res, file_contents)
2254         d.addCallback(_check2)
2255         return d
2256
2257     def test_PUT_NEWFILE_URI_only_PUT(self):
2258         d = self.PUT("/uri?t=bogus", "")
2259         d.addBoth(self.shouldFail, error.Error,
2260                   "PUT_NEWFILE_URI_only_PUT",
2261                   "400 Bad Request",
2262                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2263         return d
2264
2265     def test_PUT_NEWFILE_URI_mutable(self):
2266         file_contents = "New file contents here\n"
2267         d = self.PUT("/uri?mutable=true", file_contents)
2268         def _check_mutable(uri):
2269             uri = uri.strip()
2270             u = IURI(uri)
2271             self.failUnless(IMutableFileURI.providedBy(u))
2272             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2273             n = self.s.create_node_from_uri(uri)
2274             return n.download_best_version()
2275         d.addCallback(_check_mutable)
2276         def _check2_mutable(data):
2277             self.failUnlessEqual(data, file_contents)
2278         d.addCallback(_check2_mutable)
2279         return d
2280
2281         def _check(uri):
2282             self.failUnless(uri in FakeCHKFileNode.all_contents)
2283             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2284                                  file_contents)
2285             return self.GET("/uri/%s" % uri)
2286         d.addCallback(_check)
2287         def _check2(res):
2288             self.failUnlessEqual(res, file_contents)
2289         d.addCallback(_check2)
2290         return d
2291
2292     def test_PUT_mkdir(self):
2293         d = self.PUT("/uri?t=mkdir", "")
2294         def _check(uri):
2295             n = self.s.create_node_from_uri(uri.strip())
2296             d2 = self.failUnlessNodeKeysAre(n, [])
2297             d2.addCallback(lambda res:
2298                            self.GET("/uri/%s?t=json" % uri))
2299             return d2
2300         d.addCallback(_check)
2301         d.addCallback(self.failUnlessIsEmptyJSON)
2302         return d
2303
2304     def test_POST_check(self):
2305         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2306         def _done(res):
2307             # this returns a string form of the results, which are probably
2308             # None since we're using fake filenodes.
2309             # TODO: verify that the check actually happened, by changing
2310             # FakeCHKFileNode to count how many times .check() has been
2311             # called.
2312             pass
2313         d.addCallback(_done)
2314         return d
2315
2316     def test_bad_method(self):
2317         url = self.webish_url + self.public_url + "/foo/bar.txt"
2318         d = self.shouldHTTPError2("test_bad_method",
2319                                   501, "Not Implemented",
2320                                   "I don't know how to treat a BOGUS request.",
2321                                   client.getPage, url, method="BOGUS")
2322         return d
2323
2324     def test_short_url(self):
2325         url = self.webish_url + "/uri"
2326         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2327                                   "I don't know how to treat a DELETE request.",
2328                                   client.getPage, url, method="DELETE")
2329         return d
2330
2331     def test_ophandle_bad(self):
2332         url = self.webish_url + "/operations/bogus?t=status"
2333         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2334                                   "unknown/expired handle 'bogus'",
2335                                   client.getPage, url)
2336         return d
2337
2338     def test_ophandle_cancel(self):
2339         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2340                       followRedirect=True)
2341         d.addCallback(lambda ignored:
2342                       self.GET("/operations/128?t=status&output=JSON"))
2343         def _check1(res):
2344             data = simplejson.loads(res)
2345             self.failUnless("finished" in data, res)
2346             monitor = self.ws.root.child_operations.handles["128"][0]
2347             d = self.POST("/operations/128?t=cancel&output=JSON")
2348             def _check2(res):
2349                 data = simplejson.loads(res)
2350                 self.failUnless("finished" in data, res)
2351                 # t=cancel causes the handle to be forgotten
2352                 self.failUnless(monitor.is_cancelled())
2353             d.addCallback(_check2)
2354             return d
2355         d.addCallback(_check1)
2356         d.addCallback(lambda ignored:
2357                       self.shouldHTTPError2("test_ophandle_cancel",
2358                                             404, "404 Not Found",
2359                                             "unknown/expired handle '128'",
2360                                             self.GET,
2361                                             "/operations/128?t=status&output=JSON"))
2362         return d
2363
2364     def test_ophandle_retainfor(self):
2365         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2366                       followRedirect=True)
2367         d.addCallback(lambda ignored:
2368                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2369         def _check1(res):
2370             data = simplejson.loads(res)
2371             self.failUnless("finished" in data, res)
2372         d.addCallback(_check1)
2373         # the retain-for=0 will cause the handle to be expired very soon
2374         d.addCallback(self.stall, 2.0)
2375         d.addCallback(lambda ignored:
2376                       self.shouldHTTPError2("test_ophandle_retainfor",
2377                                             404, "404 Not Found",
2378                                             "unknown/expired handle '129'",
2379                                             self.GET,
2380                                             "/operations/129?t=status&output=JSON"))
2381         return d
2382
2383     def test_ophandle_release_after_complete(self):
2384         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2385                       followRedirect=True)
2386         d.addCallback(self.wait_for_operation, "130")
2387         d.addCallback(lambda ignored:
2388                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2389         # the release-after-complete=true will cause the handle to be expired
2390         d.addCallback(lambda ignored:
2391                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2392                                             404, "404 Not Found",
2393                                             "unknown/expired handle '130'",
2394                                             self.GET,
2395                                             "/operations/130?t=status&output=JSON"))
2396         return d
2397
2398     def test_incident(self):
2399         d = self.POST("/report_incident", details="eek")
2400         def _done(res):
2401             self.failUnless("Thank you for your report!" in res, res)
2402         d.addCallback(_done)
2403         return d
2404
2405     def test_static(self):
2406         webdir = os.path.join(self.staticdir, "subdir")
2407         fileutil.make_dirs(webdir)
2408         f = open(os.path.join(webdir, "hello.txt"), "wb")
2409         f.write("hello")
2410         f.close()
2411
2412         d = self.GET("/static/subdir/hello.txt")
2413         def _check(res):
2414             self.failUnlessEqual(res, "hello")
2415         d.addCallback(_check)
2416         return d
2417
2418
2419 class Util(unittest.TestCase):
2420     def test_abbreviate_time(self):
2421         self.failUnlessEqual(common.abbreviate_time(None), "")
2422         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2423         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2424         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2425         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2426
2427     def test_abbreviate_rate(self):
2428         self.failUnlessEqual(common.abbreviate_rate(None), "")
2429         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2430         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2431         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2432
2433     def test_abbreviate_size(self):
2434         self.failUnlessEqual(common.abbreviate_size(None), "")
2435         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2436         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2437         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2438         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2439