src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17 import common_util as testutil
  18
  19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  20 # create a webserver that works against them
  21
  22 class FakeIntroducerClient:
  23     def get_all_connectors(self):
  24         return {}
  25     def get_all_connections_for(self, service_name):
  26         return frozenset()
  27     def get_all_peerids(self):
  28         return frozenset()
  29
  30 class FakeClient(service.MultiService):
  31     nodeid = "fake_nodeid"
  32     nickname = "fake_nickname"
  33     basedir = "fake_basedir"
  34     def get_versions(self):
  35         return {'allmydata': "fake",
  36                 'foolscap': "fake",
  37                 'twisted': "fake",
  38                 'zfec': "fake",
  39                 }
  40     introducer_furl = "None"
  41     introducer_client = FakeIntroducerClient()
  42     _all_upload_status = [upload.UploadStatus()]
  43     _all_download_status = [download.DownloadStatus()]
  44     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  45     _all_publish_statuses = [publish.PublishStatus()]
  46     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  47     convergence = "some random string"
  48
  49     def connected_to_introducer(self):
  50         return False
  51
  52     def get_nickname_for_peerid(self, peerid):
  53         return u"John Doe"
  54
  55     def create_node_from_uri(self, auri):
  56         u = uri.from_string(auri)
  57         if (INewDirectoryURI.providedBy(u)
  58             or IReadonlyNewDirectoryURI.providedBy(u)):
  59             return FakeDirectoryNode(self).init_from_uri(u)
  60         if IFileURI.providedBy(u):
  61             return FakeCHKFileNode(u, self)
  62         assert IMutableFileURI.providedBy(u), u
  63         return FakeMutableFileNode(self).init_from_uri(u)
  64
  65     def create_empty_dirnode(self):
  66         n = FakeDirectoryNode(self)
  67         d = n.create()
  68         d.addCallback(lambda res: n)
  69         return d
  70
  71     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  72     def create_mutable_file(self, contents=""):
  73         n = FakeMutableFileNode(self)
  74         return n.create(contents)
  75
  76     def upload(self, uploadable):
  77         d = uploadable.get_size()
  78         d.addCallback(lambda size: uploadable.read(size))
  79         def _got_data(datav):
  80             data = "".join(datav)
  81             n = create_chk_filenode(self, data)
  82             results = upload.UploadResults()
  83             results.uri = n.get_uri()
  84             return results
  85         d.addCallback(_got_data)
  86         return d
  87
  88     def list_all_upload_statuses(self):
  89         return self._all_upload_status
  90     def list_all_download_statuses(self):
  91         return self._all_download_status
  92     def list_all_mapupdate_statuses(self):
  93         return self._all_mapupdate_statuses
  94     def list_all_publish_statuses(self):
  95         return self._all_publish_statuses
  96     def list_all_retrieve_statuses(self):
  97         return self._all_retrieve_statuses
  98     def list_all_helper_statuses(self):
  99         return []
 100
 101 class MyGetter(client.HTTPPageGetter):
 102     handleStatus_206 = lambda self: self.handleStatus_200()
 103
 104 class HTTPClientHEADFactory(client.HTTPClientFactory):
 105     protocol = MyGetter
 106
 107     def noPage(self, reason):
 108         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 109         # exception when the response to a HEAD request had no body (when in
 110         # fact they are defined to never have a body). This was fixed in
 111         # Twisted-8.0 . To work around this, we catch the
 112         # PartialDownloadError and make it disappear.
 113         if (reason.check(client.PartialDownloadError)
 114             and self.method.upper() == "HEAD"):
 115             self.page("")
 116             return
 117         return client.HTTPClientFactory.noPage(self, reason)
 118
 119 class HTTPClientGETFactory(client.HTTPClientFactory):
 120     protocol = MyGetter
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 128         s.setServiceParent(self.s)
 129         self.webish_port = port = s.listener._port.getHost().port
 130         self.webish_url = "http://localhost:%d" % port
 131
 132         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 133         d = defer.DeferredList(l)
 134         def _then(res):
 135             self.public_root = res[0][1]
 136             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 137             self.public_url = "/uri/" + self.public_root.get_uri()
 138             self.private_root = res[1][1]
 139
 140             foo = res[2][1]
 141             self._foo_node = foo
 142             self._foo_uri = foo.get_uri()
 143             self._foo_readonly_uri = foo.get_readonly_uri()
 144             # NOTE: we ignore the deferred on all set_uri() calls, because we
 145             # know the fake nodes do these synchronously
 146             self.public_root.set_uri(u"foo", foo.get_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 150
 151             foo.set_uri(u"empty", res[3][1].get_uri())
 152             sub_uri = res[4][1].get_uri()
 153             self._sub_uri = sub_uri
 154             foo.set_uri(u"sub", sub_uri)
 155             sub = self.s.create_node_from_uri(sub_uri)
 156
 157             _ign, n, blocking_uri = self.makefile(1)
 158             foo.set_uri(u"blockingfile", blocking_uri)
 159
 160             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 161             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 162             # still think of it as an umlaut
 163             foo.set_uri(unicode_filename, self._bar_txt_uri)
 164
 165             _ign, n, baz_file = self.makefile(2)
 166             sub.set_uri(u"baz.txt", baz_file)
 167
 168             _ign, n, self._bad_file_uri = self.makefile(3)
 169             # this uri should not be downloadable
 170             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 171
 172             rodir = res[5][1]
 173             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(self.s, contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 214
 215     def failUnlessIsFooJSON(self, res):
 216         data = simplejson.loads(res)
 217         self.failUnless(isinstance(data, list))
 218         self.failUnlessEqual(data[0], "dirnode", res)
 219         self.failUnless(isinstance(data[1], dict))
 220         self.failUnless(data[1]["mutable"])
 221         self.failUnless("rw_uri" in data[1]) # mutable
 222         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 223         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 224
 225         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 226         self.failUnlessEqual(kidnames,
 227                              [u"bar.txt", u"blockingfile", u"empty",
 228                               u"n\u00fc.txt", u"sub"])
 229         kids = dict( [(unicode(name),value)
 230                       for (name,value)
 231                       in data[1]["children"].iteritems()] )
 232         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 233         self.failUnless("metadata" in kids[u"sub"][1])
 234         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 235         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 236         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 237         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 238         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 240                              self._bar_txt_metadata["ctime"])
 241         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 242                              self._bar_txt_uri)
 243
 244     def GET(self, urlpath, followRedirect=False, return_response=False,
 245             **kwargs):
 246         # if return_response=True, this fires with (data, statuscode,
 247         # respheaders) instead of just data.
 248         assert not isinstance(urlpath, unicode)
 249         url = self.webish_url + urlpath
 250         factory = HTTPClientGETFactory(url, method="GET",
 251                                        followRedirect=followRedirect, **kwargs)
 252         reactor.connectTCP("localhost", self.webish_port, factory)
 253         d = factory.deferred
 254         def _got_data(data):
 255             return (data, factory.status, factory.response_headers)
 256         if return_response:
 257             d.addCallback(_got_data)
 258         return factory.deferred
 259
 260     def HEAD(self, urlpath, return_response=False, **kwargs):
 261         # this requires some surgery, because twisted.web.client doesn't want
 262         # to give us back the response headers.
 263         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 264         reactor.connectTCP("localhost", self.webish_port, factory)
 265         d = factory.deferred
 266         def _got_data(data):
 267             return (data, factory.status, factory.response_headers)
 268         if return_response:
 269             d.addCallback(_got_data)
 270         return factory.deferred
 271
 272     def PUT(self, urlpath, data, **kwargs):
 273         url = self.webish_url + urlpath
 274         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 275
 276     def DELETE(self, urlpath):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="DELETE")
 279
 280     def POST(self, urlpath, followRedirect=False, **fields):
 281         url = self.webish_url + urlpath
 282         sepbase = "boogabooga"
 283         sep = "--" + sepbase
 284         form = []
 285         form.append(sep)
 286         form.append('Content-Disposition: form-data; name="_charset"')
 287         form.append('')
 288         form.append('UTF-8')
 289         form.append(sep)
 290         for name, value in fields.iteritems():
 291             if isinstance(value, tuple):
 292                 filename, value = value
 293                 form.append('Content-Disposition: form-data; name="%s"; '
 294                             'filename="%s"' % (name, filename.encode("utf-8")))
 295             else:
 296                 form.append('Content-Disposition: form-data; name="%s"' % name)
 297             form.append('')
 298             if isinstance(value, unicode):
 299                 value = value.encode("utf-8")
 300             else:
 301                 value = str(value)
 302             assert isinstance(value, str)
 303             form.append(value)
 304             form.append(sep)
 305         form[-1] += "--"
 306         body = "\r\n".join(form) + "\r\n"
 307         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 308                    }
 309         return client.getPage(url, method="POST", postdata=body,
 310                               headers=headers, followRedirect=followRedirect)
 311
 312     def shouldFail(self, res, expected_failure, which,
 313                    substring=None, response_substring=None):
 314         if isinstance(res, failure.Failure):
 315             res.trap(expected_failure)
 316             if substring:
 317                 self.failUnless(substring in str(res),
 318                                 "substring '%s' not in '%s'"
 319                                 % (substring, str(res)))
 320             if response_substring:
 321                 self.failUnless(response_substring in res.value.response,
 322                                 "response substring '%s' not in '%s'"
 323                                 % (response_substring, res.value.response))
 324         else:
 325             self.fail("%s was supposed to raise %s, not get '%s'" %
 326                       (which, expected_failure, res))
 327
 328     def shouldFail2(self, expected_failure, which, substring,
 329                     response_substring,
 330                     callable, *args, **kwargs):
 331         assert substring is None or isinstance(substring, str)
 332         assert response_substring is None or isinstance(response_substring, str)
 333         d = defer.maybeDeferred(callable, *args, **kwargs)
 334         def done(res):
 335             if isinstance(res, failure.Failure):
 336                 res.trap(expected_failure)
 337                 if substring:
 338                     self.failUnless(substring in str(res),
 339                                     "%s: substring '%s' not in '%s'"
 340                                     % (which, substring, str(res)))
 341                 if response_substring:
 342                     self.failUnless(response_substring in res.value.response,
 343                                     "%s: response substring '%s' not in '%s'"
 344                                     % (which,
 345                                        response_substring, res.value.response))
 346             else:
 347                 self.fail("%s was supposed to raise %s, not get '%s'" %
 348                           (which, expected_failure, res))
 349         d.addBoth(done)
 350         return d
 351
 352     def should404(self, res, which):
 353         if isinstance(res, failure.Failure):
 354             res.trap(error.Error)
 355             self.failUnlessEqual(res.value.status, "404")
 356         else:
 357             self.fail("%s was supposed to Error(404), not get '%s'" %
 358                       (which, res))
 359
 360     def shouldHTTPError(self, res, which, code=None, substring=None,
 361                         response_substring=None):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             if code is not None:
 365                 self.failUnlessEqual(res.value.status, str(code))
 366             if substring:
 367                 self.failUnless(substring in str(res),
 368                                 "substring '%s' not in '%s'"
 369                                 % (substring, str(res)))
 370             if response_substring:
 371                 self.failUnless(response_substring in res.value.response,
 372                                 "response substring '%s' not in '%s'"
 373                                 % (response_substring, res.value.response))
 374         else:
 375             self.fail("%s was supposed to Error(%s), not get '%s'" %
 376                       (which, code, res))
 377
 378     def shouldHTTPError2(self, which,
 379                          code=None, substring=None, response_substring=None,
 380                          callable=None, *args, **kwargs):
 381         assert substring is None or isinstance(substring, str)
 382         assert callable
 383         d = defer.maybeDeferred(callable, *args, **kwargs)
 384         d.addBoth(self.shouldHTTPError, which,
 385                   code, substring, response_substring)
 386         return d
 387
 388
 389 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 390     def test_create(self):
 391         pass
 392
 393     def test_welcome(self):
 394         d = self.GET("/")
 395         def _check(res):
 396             self.failUnless('Welcome To AllMyData' in res)
 397             self.failUnless('Tahoe' in res)
 398
 399             self.s.basedir = 'web/test_welcome'
 400             fileutil.make_dirs("web/test_welcome")
 401             fileutil.make_dirs("web/test_welcome/private")
 402             return self.GET("/")
 403         d.addCallback(_check)
 404         return d
 405
 406     def test_provisioning_math(self):
 407         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 408         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 409         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 410         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 411         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 412
 413     def test_provisioning(self):
 414         d = self.GET("/provisioning/")
 415         def _check(res):
 416             self.failUnless('Tahoe Provisioning Tool' in res)
 417             fields = {'filled': True,
 418                       "num_users": int(50e3),
 419                       "files_per_user": 1000,
 420                       "space_per_user": int(1e9),
 421                       "sharing_ratio": 1.0,
 422                       "encoding_parameters": "3-of-10-5",
 423                       "num_servers": 30,
 424                       "ownership_mode": "A",
 425                       "download_rate": 100,
 426                       "upload_rate": 10,
 427                       "delete_rate": 10,
 428                       "lease_timer": 7,
 429                       }
 430             return self.POST("/provisioning/", **fields)
 431
 432         d.addCallback(_check)
 433         def _check2(res):
 434             self.failUnless('Tahoe Provisioning Tool' in res)
 435             self.failUnless("Share space consumed: 167.01TB" in res)
 436
 437             fields = {'filled': True,
 438                       "num_users": int(50e6),
 439                       "files_per_user": 1000,
 440                       "space_per_user": int(5e9),
 441                       "sharing_ratio": 1.0,
 442                       "encoding_parameters": "25-of-100-50",
 443                       "num_servers": 30000,
 444                       "ownership_mode": "E",
 445                       "drive_failure_model": "U",
 446                       "drive_size": 1000,
 447                       "download_rate": 1000,
 448                       "upload_rate": 100,
 449                       "delete_rate": 100,
 450                       "lease_timer": 7,
 451                       }
 452             return self.POST("/provisioning/", **fields)
 453         d.addCallback(_check2)
 454         def _check3(res):
 455             self.failUnless("Share space consumed: huge!" in res)
 456             fields = {'filled': True}
 457             return self.POST("/provisioning/", **fields)
 458         d.addCallback(_check3)
 459         def _check4(res):
 460             self.failUnless("Share space consumed:" in res)
 461         d.addCallback(_check4)
 462         return d
 463
 464     def test_status(self):
 465         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 466         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 467         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 468         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 469         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 470         d = self.GET("/status", followRedirect=True)
 471         def _check(res):
 472             self.failUnless('Upload and Download Status' in res, res)
 473             self.failUnless('"down-%d"' % dl_num in res, res)
 474             self.failUnless('"up-%d"' % ul_num in res, res)
 475             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 476             self.failUnless('"publish-%d"' % pub_num in res, res)
 477             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 478         d.addCallback(_check)
 479         d.addCallback(lambda res: self.GET("/status/?t=json"))
 480         def _check_json(res):
 481             data = simplejson.loads(res)
 482             self.failUnless(isinstance(data, dict))
 483             active = data["active"]
 484             # TODO: test more. We need a way to fake an active operation
 485             # here.
 486         d.addCallback(_check_json)
 487
 488         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 489         def _check_dl(res):
 490             self.failUnless("File Download Status" in res, res)
 491         d.addCallback(_check_dl)
 492         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 493         def _check_ul(res):
 494             self.failUnless("File Upload Status" in res, res)
 495         d.addCallback(_check_ul)
 496         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 497         def _check_mapupdate(res):
 498             self.failUnless("Mutable File Servermap Update Status" in res, res)
 499         d.addCallback(_check_mapupdate)
 500         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 501         def _check_publish(res):
 502             self.failUnless("Mutable File Publish Status" in res, res)
 503         d.addCallback(_check_publish)
 504         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 505         def _check_retrieve(res):
 506             self.failUnless("Mutable File Retrieve Status" in res, res)
 507         d.addCallback(_check_retrieve)
 508
 509         return d
 510
 511     def test_status_numbers(self):
 512         drrm = status.DownloadResultsRendererMixin()
 513         self.failUnlessEqual(drrm.render_time(None, None), "")
 514         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 515         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 517         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 518         self.failUnlessEqual(drrm.render_rate(None, None), "")
 519         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 521         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 522
 523         urrm = status.UploadResultsRendererMixin()
 524         self.failUnlessEqual(urrm.render_time(None, None), "")
 525         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 526         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 528         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 529         self.failUnlessEqual(urrm.render_rate(None, None), "")
 530         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 532         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 533
 534     def test_GET_FILEURL(self):
 535         d = self.GET(self.public_url + "/foo/bar.txt")
 536         d.addCallback(self.failUnlessIsBarDotTxt)
 537         return d
 538
 539     def test_GET_FILEURL_range(self):
 540         headers = {"range": "bytes=1-10"}
 541         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 542                      return_response=True)
 543         def _got((res, status, headers)):
 544             self.failUnlessEqual(int(status), 206)
 545             self.failUnless(headers.has_key("content-range"))
 546             self.failUnlessEqual(headers["content-range"][0],
 547                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 548             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 549         d.addCallback(_got)
 550         return d
 551
 552     def test_HEAD_FILEURL_range(self):
 553         headers = {"range": "bytes=1-10"}
 554         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 555                      return_response=True)
 556         def _got((res, status, headers)):
 557             self.failUnlessEqual(res, "")
 558             self.failUnlessEqual(int(status), 206)
 559             self.failUnless(headers.has_key("content-range"))
 560             self.failUnlessEqual(headers["content-range"][0],
 561                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 562         d.addCallback(_got)
 563         return d
 564
 565     def test_GET_FILEURL_range_bad(self):
 566         headers = {"range": "BOGUS=fizbop-quarnak"}
 567         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 568                              "400 Bad Request",
 569                              "Syntactically invalid http range header",
 570                              self.GET, self.public_url + "/foo/bar.txt",
 571                              headers=headers)
 572         return d
 573
 574     def test_HEAD_FILEURL(self):
 575         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 576         def _got((res, status, headers)):
 577             self.failUnlessEqual(res, "")
 578             self.failUnlessEqual(headers["content-length"][0],
 579                                  str(len(self.BAR_CONTENTS)))
 580             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 581         d.addCallback(_got)
 582         return d
 583
 584     def test_GET_FILEURL_named(self):
 585         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 586         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 587         d = self.GET(base + "/@@name=/blah.txt")
 588         d.addCallback(self.failUnlessIsBarDotTxt)
 589         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 590         d.addCallback(self.failUnlessIsBarDotTxt)
 591         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 592         d.addCallback(self.failUnlessIsBarDotTxt)
 593         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 594         d.addCallback(self.failUnlessIsBarDotTxt)
 595         save_url = base + "?save=true&filename=blah.txt"
 596         d.addCallback(lambda res: self.GET(save_url))
 597         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 598         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 599         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 600         u_url = base + "?save=true&filename=" + u_fn_e
 601         d.addCallback(lambda res: self.GET(u_url))
 602         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 603         return d
 604
 605     def test_PUT_FILEURL_named_bad(self):
 606         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 607         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 608                              "400 Bad Request",
 609                              "/file can only be used with GET or HEAD",
 610                              self.PUT, base + "/@@name=/blah.txt", "")
 611         return d
 612
 613     def test_GET_DIRURL_named_bad(self):
 614         base = "/file/%s" % urllib.quote(self._foo_uri)
 615         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 616                              "400 Bad Request",
 617                              "is not a file-cap",
 618                              self.GET, base + "/@@name=/blah.txt")
 619         return d
 620
 621     def test_GET_slash_file_bad(self):
 622         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 623                              "404 Not Found",
 624                              "/file must be followed by a file-cap and a name",
 625                              self.GET, "/file")
 626         return d
 627
 628     def test_GET_unhandled_URI_named(self):
 629         contents, n, newuri = self.makefile(12)
 630         verifier_cap = n.get_verifier().to_string()
 631         base = "/file/%s" % urllib.quote(verifier_cap)
 632         # client.create_node_from_uri() can't handle verify-caps
 633         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 634                              "400 Bad Request",
 635                              "is not a valid file- or directory- cap",
 636                              self.GET, base)
 637         return d
 638
 639     def test_GET_unhandled_URI(self):
 640         contents, n, newuri = self.makefile(12)
 641         verifier_cap = n.get_verifier().to_string()
 642         base = "/uri/%s" % urllib.quote(verifier_cap)
 643         # client.create_node_from_uri() can't handle verify-caps
 644         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 645                              "400 Bad Request",
 646                              "is not a valid file- or directory- cap",
 647                              self.GET, base)
 648         return d
 649
 650     def test_GET_FILE_URI(self):
 651         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 652         d = self.GET(base)
 653         d.addCallback(self.failUnlessIsBarDotTxt)
 654         return d
 655
 656     def test_GET_FILE_URI_badchild(self):
 657         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 658         errmsg = "Files have no children, certainly not named 'boguschild'"
 659         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 660                              "400 Bad Request", errmsg,
 661                              self.GET, base)
 662         return d
 663
 664     def test_PUT_FILE_URI_badchild(self):
 665         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 666         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 667         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 668                              "400 Bad Request", errmsg,
 669                              self.PUT, base, "")
 670         return d
 671
 672     def test_GET_FILEURL_save(self):
 673         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 674         # TODO: look at the headers, expect a Content-Disposition: attachment
 675         # header.
 676         d.addCallback(self.failUnlessIsBarDotTxt)
 677         return d
 678
 679     def test_GET_FILEURL_missing(self):
 680         d = self.GET(self.public_url + "/foo/missing")
 681         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 682         return d
 683
 684     def test_PUT_NEWFILEURL(self):
 685         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 686         # TODO: we lose the response code, so we can't check this
 687         #self.failUnlessEqual(responsecode, 201)
 688         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 689         d.addCallback(lambda res:
 690                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 691                                                       self.NEWFILE_CONTENTS))
 692         return d
 693
 694     def test_PUT_NEWFILEURL_range_bad(self):
 695         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 696         target = self.public_url + "/foo/new.txt"
 697         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 698                              "501 Not Implemented",
 699                              "Content-Range in PUT not yet supported",
 700                              # (and certainly not for immutable files)
 701                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 702                              headers=headers)
 703         d.addCallback(lambda res:
 704                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 705         return d
 706
 707     def test_PUT_NEWFILEURL_mutable(self):
 708         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 709                      self.NEWFILE_CONTENTS)
 710         # TODO: we lose the response code, so we can't check this
 711         #self.failUnlessEqual(responsecode, 201)
 712         def _check_uri(res):
 713             u = uri.from_string_mutable_filenode(res)
 714             self.failUnless(u.is_mutable())
 715             self.failIf(u.is_readonly())
 716             return res
 717         d.addCallback(_check_uri)
 718         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 719         d.addCallback(lambda res:
 720                       self.failUnlessMutableChildContentsAre(self._foo_node,
 721                                                              u"new.txt",
 722                                                              self.NEWFILE_CONTENTS))
 723         return d
 724
 725     def test_PUT_NEWFILEURL_mutable_toobig(self):
 726         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 727                              "413 Request Entity Too Large",
 728                              "SDMF is limited to one segment, and 10001 > 10000",
 729                              self.PUT,
 730                              self.public_url + "/foo/new.txt?mutable=true",
 731                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 732         return d
 733
 734     def test_PUT_NEWFILEURL_replace(self):
 735         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 200)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_bad_t(self):
 745         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 746                              "PUT to a file: bad t=bogus",
 747                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 748                              "contents")
 749         return d
 750
 751     def test_PUT_NEWFILEURL_no_replace(self):
 752         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 753                      self.NEWFILE_CONTENTS)
 754         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 755                   "409 Conflict",
 756                   "There was already a child by that name, and you asked me "
 757                   "to not replace it")
 758         return d
 759
 760     def test_PUT_NEWFILEURL_mkdirs(self):
 761         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 762         fn = self._foo_node
 763         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 764         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 765         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_blocked(self):
 772         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 773                      self.NEWFILE_CONTENTS)
 774         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 775                   "409 Conflict",
 776                   "Unable to create directory 'blockingfile': a file was in the way")
 777         return d
 778
 779     def test_DELETE_FILEURL(self):
 780         d = self.DELETE(self.public_url + "/foo/bar.txt")
 781         d.addCallback(lambda res:
 782                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 783         return d
 784
 785     def test_DELETE_FILEURL_missing(self):
 786         d = self.DELETE(self.public_url + "/foo/missing")
 787         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 788         return d
 789
 790     def test_DELETE_FILEURL_missing2(self):
 791         d = self.DELETE(self.public_url + "/missing/missing")
 792         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 793         return d
 794
 795     def test_GET_FILEURL_json(self):
 796         # twisted.web.http.parse_qs ignores any query args without an '=', so
 797         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 798         # instead. This may make it tricky to emulate the S3 interface
 799         # completely.
 800         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 801         d.addCallback(self.failUnlessIsBarJSON)
 802         return d
 803
 804     def test_GET_FILEURL_json_missing(self):
 805         d = self.GET(self.public_url + "/foo/missing?json")
 806         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 807         return d
 808
 809     def test_GET_FILEURL_uri(self):
 810         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 811         def _check(res):
 812             self.failUnlessEqual(res, self._bar_txt_uri)
 813         d.addCallback(_check)
 814         d.addCallback(lambda res:
 815                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 816         def _check2(res):
 817             # for now, for files, uris and readonly-uris are the same
 818             self.failUnlessEqual(res, self._bar_txt_uri)
 819         d.addCallback(_check2)
 820         return d
 821
 822     def test_GET_FILEURL_badtype(self):
 823         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 824                                   "bad t=bogus",
 825                                   self.GET,
 826                                   self.public_url + "/foo/bar.txt?t=bogus")
 827         return d
 828
 829     def test_GET_FILEURL_uri_missing(self):
 830         d = self.GET(self.public_url + "/foo/missing?t=uri")
 831         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 832         return d
 833
 834     def test_GET_DIRURL(self):
 835         # the addSlash means we get a redirect here
 836         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 837         ROOT = "../../.."
 838         d = self.GET(self.public_url + "/foo", followRedirect=True)
 839         def _check(res):
 840             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 841                             in res, res)
 842             # the FILE reference points to a URI, but it should end in bar.txt
 843             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 844                        (ROOT, urllib.quote(self._bar_txt_uri)))
 845             get_bar = "".join([r'<td>',
 846                                r'<a href="%s">bar.txt</a>' % bar_url,
 847                                r'</td>',
 848                                r'\s+<td>FILE</td>',
 849                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 850                                ])
 851             self.failUnless(re.search(get_bar, res), res)
 852             for line in res.split("\n"):
 853                 # find the line that contains the delete button for bar.txt
 854                 if ("form action" in line and
 855                     'value="delete"' in line and
 856                     'value="bar.txt"' in line):
 857                     # the form target should use a relative URL
 858                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 859                     self.failUnless(('action="%s"' % foo_url) in line, line)
 860                     # and the when_done= should too
 861                     #done_url = urllib.quote(???)
 862                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 863                     #                in line, line)
 864                     break
 865             else:
 866                 self.fail("unable to find delete-bar.txt line", res)
 867
 868             # the DIR reference just points to a URI
 869             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 870             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 871                        + r'\s+<td>DIR</td>')
 872             self.failUnless(re.search(get_sub, res), res)
 873         d.addCallback(_check)
 874
 875         # look at a directory which is readonly
 876         d.addCallback(lambda res:
 877                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 878         def _check2(res):
 879             self.failUnless("(readonly)" in res, res)
 880             self.failIf("Upload a file" in res, res)
 881         d.addCallback(_check2)
 882
 883         # and at a directory that contains a readonly directory
 884         d.addCallback(lambda res:
 885                       self.GET(self.public_url, followRedirect=True))
 886         def _check3(res):
 887             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 888                                       '</td>\s+<td>DIR-RO</td>', res))
 889         d.addCallback(_check3)
 890
 891         return d
 892
 893     def test_GET_DIRURL_badtype(self):
 894         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 895                                   400, "Bad Request",
 896                                   "bad t=bogus",
 897                                   self.GET,
 898                                   self.public_url + "/foo?t=bogus")
 899         return d
 900
 901     def test_GET_DIRURL_json(self):
 902         d = self.GET(self.public_url + "/foo?t=json")
 903         d.addCallback(self.failUnlessIsFooJSON)
 904         return d
 905
 906
 907     def test_POST_DIRURL_manifest_no_ophandle(self):
 908         d = self.shouldFail2(error.Error,
 909                              "test_POST_DIRURL_manifest_no_ophandle",
 910                              "400 Bad Request",
 911                              "slow operation requires ophandle=",
 912                              self.POST, self.public_url, t="start-manifest")
 913         return d
 914
 915     def test_POST_DIRURL_manifest(self):
 916         d = defer.succeed(None)
 917         def getman(ignored, output):
 918             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 919                           followRedirect=True)
 920             d.addCallback(self.wait_for_operation, "125")
 921             d.addCallback(self.get_operation_results, "125", output)
 922             return d
 923         d.addCallback(getman, None)
 924         def _got_html(manifest):
 925             self.failUnless("Manifest of SI=" in manifest)
 926             self.failUnless("<td>sub</td>" in manifest)
 927             self.failUnless(self._sub_uri in manifest)
 928             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 929         d.addCallback(_got_html)
 930
 931         # both t=status and unadorned GET should be identical
 932         d.addCallback(lambda res: self.GET("/operations/125"))
 933         d.addCallback(_got_html)
 934
 935         d.addCallback(getman, "html")
 936         d.addCallback(_got_html)
 937         d.addCallback(getman, "text")
 938         def _got_text(manifest):
 939             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 940             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 941         d.addCallback(_got_text)
 942         d.addCallback(getman, "JSON")
 943         def _got_json(res):
 944             data = res["manifest"]
 945             got = {}
 946             for (path_list, cap) in data:
 947                 got[tuple(path_list)] = cap
 948             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 949             self.failUnless((u"sub",u"baz.txt") in got)
 950             self.failUnless("finished" in res)
 951             self.failUnless("origin" in res)
 952             self.failUnless("storage-index" in res)
 953             self.failUnless("stats" in res)
 954         d.addCallback(_got_json)
 955         return d
 956
 957     def test_POST_DIRURL_deepsize_no_ophandle(self):
 958         d = self.shouldFail2(error.Error,
 959                              "test_POST_DIRURL_deepsize_no_ophandle",
 960                              "400 Bad Request",
 961                              "slow operation requires ophandle=",
 962                              self.POST, self.public_url, t="start-deep-size")
 963         return d
 964
 965     def test_POST_DIRURL_deepsize(self):
 966         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 967                       followRedirect=True)
 968         d.addCallback(self.wait_for_operation, "126")
 969         d.addCallback(self.get_operation_results, "126", "json")
 970         def _got_json(data):
 971             self.failUnlessEqual(data["finished"], True)
 972             size = data["size"]
 973             self.failUnless(size > 1000)
 974         d.addCallback(_got_json)
 975         d.addCallback(self.get_operation_results, "126", "text")
 976         def _got_text(res):
 977             mo = re.search(r'^size: (\d+)$', res, re.M)
 978             self.failUnless(mo, res)
 979             size = int(mo.group(1))
 980             # with directories, the size varies.
 981             self.failUnless(size > 1000)
 982         d.addCallback(_got_text)
 983         return d
 984
 985     def test_POST_DIRURL_deepstats_no_ophandle(self):
 986         d = self.shouldFail2(error.Error,
 987                              "test_POST_DIRURL_deepstats_no_ophandle",
 988                              "400 Bad Request",
 989                              "slow operation requires ophandle=",
 990                              self.POST, self.public_url, t="start-deep-stats")
 991         return d
 992
 993     def test_POST_DIRURL_deepstats(self):
 994         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 995                       followRedirect=True)
 996         d.addCallback(self.wait_for_operation, "127")
 997         d.addCallback(self.get_operation_results, "127", "json")
 998         def _got_json(stats):
 999             expected = {"count-immutable-files": 3,
1000                         "count-mutable-files": 0,
1001                         "count-literal-files": 0,
1002                         "count-files": 3,
1003                         "count-directories": 3,
1004                         "size-immutable-files": 57,
1005                         "size-literal-files": 0,
1006                         #"size-directories": 1912, # varies
1007                         #"largest-directory": 1590,
1008                         "largest-directory-children": 5,
1009                         "largest-immutable-file": 19,
1010                         }
1011             for k,v in expected.iteritems():
1012                 self.failUnlessEqual(stats[k], v,
1013                                      "stats[%s] was %s, not %s" %
1014                                      (k, stats[k], v))
1015             self.failUnlessEqual(stats["size-files-histogram"],
1016                                  [ [11, 31, 3] ])
1017         d.addCallback(_got_json)
1018         return d
1019
1020     def test_GET_DIRURL_uri(self):
1021         d = self.GET(self.public_url + "/foo?t=uri")
1022         def _check(res):
1023             self.failUnlessEqual(res, self._foo_uri)
1024         d.addCallback(_check)
1025         return d
1026
1027     def test_GET_DIRURL_readonly_uri(self):
1028         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1029         def _check(res):
1030             self.failUnlessEqual(res, self._foo_readonly_uri)
1031         d.addCallback(_check)
1032         return d
1033
1034     def test_PUT_NEWDIRURL(self):
1035         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1036         d.addCallback(lambda res:
1037                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1038         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1039         d.addCallback(self.failUnlessNodeKeysAre, [])
1040         return d
1041
1042     def test_PUT_NEWDIRURL_exists(self):
1043         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1044         d.addCallback(lambda res:
1045                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1046         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1047         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1048         return d
1049
1050     def test_PUT_NEWDIRURL_blocked(self):
1051         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1052                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1053                              self.PUT,
1054                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1055         d.addCallback(lambda res:
1056                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1057         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1058         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1059         return d
1060
1061     def test_PUT_NEWDIRURL_mkdir_p(self):
1062         d = defer.succeed(None)
1063         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1064         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1065         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1066         def mkdir_p(mkpnode):
1067             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1068             d = self.POST(url)
1069             def made_subsub(ssuri):
1070                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1071                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1072                 d = self.POST(url)
1073                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1074                 return d
1075             d.addCallback(made_subsub)
1076             return d
1077         d.addCallback(mkdir_p)
1078         return d
1079
1080     def test_PUT_NEWDIRURL_mkdirs(self):
1081         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1082         d.addCallback(lambda res:
1083                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1084         d.addCallback(lambda res:
1085                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1086         d.addCallback(lambda res:
1087                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1088         d.addCallback(self.failUnlessNodeKeysAre, [])
1089         return d
1090
1091     def test_DELETE_DIRURL(self):
1092         d = self.DELETE(self.public_url + "/foo")
1093         d.addCallback(lambda res:
1094                       self.failIfNodeHasChild(self.public_root, u"foo"))
1095         return d
1096
1097     def test_DELETE_DIRURL_missing(self):
1098         d = self.DELETE(self.public_url + "/foo/missing")
1099         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1100         d.addCallback(lambda res:
1101                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1102         return d
1103
1104     def test_DELETE_DIRURL_missing2(self):
1105         d = self.DELETE(self.public_url + "/missing")
1106         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1107         return d
1108
1109     def dump_root(self):
1110         print "NODEWALK"
1111         w = webish.DirnodeWalkerMixin()
1112         def visitor(childpath, childnode, metadata):
1113             print childpath
1114         d = w.walk(self.public_root, visitor)
1115         return d
1116
1117     def failUnlessNodeKeysAre(self, node, expected_keys):
1118         for k in expected_keys:
1119             assert isinstance(k, unicode)
1120         d = node.list()
1121         def _check(children):
1122             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1123         d.addCallback(_check)
1124         return d
1125     def failUnlessNodeHasChild(self, node, name):
1126         assert isinstance(name, unicode)
1127         d = node.list()
1128         def _check(children):
1129             self.failUnless(name in children)
1130         d.addCallback(_check)
1131         return d
1132     def failIfNodeHasChild(self, node, name):
1133         assert isinstance(name, unicode)
1134         d = node.list()
1135         def _check(children):
1136             self.failIf(name in children)
1137         d.addCallback(_check)
1138         return d
1139
1140     def failUnlessChildContentsAre(self, node, name, expected_contents):
1141         assert isinstance(name, unicode)
1142         d = node.get_child_at_path(name)
1143         d.addCallback(lambda node: node.download_to_data())
1144         def _check(contents):
1145             self.failUnlessEqual(contents, expected_contents)
1146         d.addCallback(_check)
1147         return d
1148
1149     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1150         assert isinstance(name, unicode)
1151         d = node.get_child_at_path(name)
1152         d.addCallback(lambda node: node.download_best_version())
1153         def _check(contents):
1154             self.failUnlessEqual(contents, expected_contents)
1155         d.addCallback(_check)
1156         return d
1157
1158     def failUnlessChildURIIs(self, node, name, expected_uri):
1159         assert isinstance(name, unicode)
1160         d = node.get_child_at_path(name)
1161         def _check(child):
1162             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1163         d.addCallback(_check)
1164         return d
1165
1166     def failUnlessURIMatchesChild(self, got_uri, node, name):
1167         assert isinstance(name, unicode)
1168         d = node.get_child_at_path(name)
1169         def _check(child):
1170             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1171         d.addCallback(_check)
1172         return d
1173
1174     def failUnlessCHKURIHasContents(self, got_uri, contents):
1175         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1176
1177     def test_POST_upload(self):
1178         d = self.POST(self.public_url + "/foo", t="upload",
1179                       file=("new.txt", self.NEWFILE_CONTENTS))
1180         fn = self._foo_node
1181         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1182         d.addCallback(lambda res:
1183                       self.failUnlessChildContentsAre(fn, u"new.txt",
1184                                                       self.NEWFILE_CONTENTS))
1185         return d
1186
1187     def test_POST_upload_unicode(self):
1188         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1189         d = self.POST(self.public_url + "/foo", t="upload",
1190                       file=(filename, self.NEWFILE_CONTENTS))
1191         fn = self._foo_node
1192         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1193         d.addCallback(lambda res:
1194                       self.failUnlessChildContentsAre(fn, filename,
1195                                                       self.NEWFILE_CONTENTS))
1196         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1197         d.addCallback(lambda res: self.GET(target_url))
1198         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1199                                                             self.NEWFILE_CONTENTS,
1200                                                             contents))
1201         return d
1202
1203     def test_POST_upload_unicode_named(self):
1204         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1205         d = self.POST(self.public_url + "/foo", t="upload",
1206                       name=filename,
1207                       file=("overridden", self.NEWFILE_CONTENTS))
1208         fn = self._foo_node
1209         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1210         d.addCallback(lambda res:
1211                       self.failUnlessChildContentsAre(fn, filename,
1212                                                       self.NEWFILE_CONTENTS))
1213         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1214         d.addCallback(lambda res: self.GET(target_url))
1215         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1216                                                             self.NEWFILE_CONTENTS,
1217                                                             contents))
1218         return d
1219
1220     def test_POST_upload_no_link(self):
1221         d = self.POST("/uri", t="upload",
1222                       file=("new.txt", self.NEWFILE_CONTENTS))
1223         def _check_upload_results(page):
1224             # this should be a page which describes the results of the upload
1225             # that just finished.
1226             self.failUnless("Upload Results:" in page)
1227             self.failUnless("URI:" in page)
1228             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1229             mo = uri_re.search(page)
1230             self.failUnless(mo, page)
1231             new_uri = mo.group(1)
1232             return new_uri
1233         d.addCallback(_check_upload_results)
1234         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1235         return d
1236
1237     def test_POST_upload_no_link_whendone(self):
1238         d = self.POST("/uri", t="upload", when_done="/",
1239                       file=("new.txt", self.NEWFILE_CONTENTS))
1240         d.addBoth(self.shouldRedirect, "/")
1241         return d
1242
1243     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1244         d = defer.maybeDeferred(callable, *args, **kwargs)
1245         def done(res):
1246             if isinstance(res, failure.Failure):
1247                 res.trap(error.PageRedirect)
1248                 statuscode = res.value.status
1249                 target = res.value.location
1250                 return checker(statuscode, target)
1251             self.fail("%s: callable was supposed to redirect, not return '%s'"
1252                       % (which, res))
1253         d.addBoth(done)
1254         return d
1255
1256     def test_POST_upload_no_link_whendone_results(self):
1257         def check(statuscode, target):
1258             self.failUnlessEqual(statuscode, str(http.FOUND))
1259             self.failUnless(target.startswith(self.webish_url), target)
1260             return client.getPage(target, method="GET")
1261         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1262                                  check,
1263                                  self.POST, "/uri", t="upload",
1264                                  when_done="/uri/%(uri)s",
1265                                  file=("new.txt", self.NEWFILE_CONTENTS))
1266         d.addCallback(lambda res:
1267                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1268         return d
1269
1270     def test_POST_upload_no_link_mutable(self):
1271         d = self.POST("/uri", t="upload", mutable="true",
1272                       file=("new.txt", self.NEWFILE_CONTENTS))
1273         def _check(new_uri):
1274             new_uri = new_uri.strip()
1275             self.new_uri = new_uri
1276             u = IURI(new_uri)
1277             self.failUnless(IMutableFileURI.providedBy(u))
1278             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1279             n = self.s.create_node_from_uri(new_uri)
1280             return n.download_best_version()
1281         d.addCallback(_check)
1282         def _check2(data):
1283             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1284             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1285         d.addCallback(_check2)
1286         def _check3(data):
1287             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1288             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1289         d.addCallback(_check3)
1290         def _check4(data):
1291             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1292         d.addCallback(_check4)
1293         return d
1294
1295     def test_POST_upload_no_link_mutable_toobig(self):
1296         d = self.shouldFail2(error.Error,
1297                              "test_POST_upload_no_link_mutable_toobig",
1298                              "413 Request Entity Too Large",
1299                              "SDMF is limited to one segment, and 10001 > 10000",
1300                              self.POST,
1301                              "/uri", t="upload", mutable="true",
1302                              file=("new.txt",
1303                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1304         return d
1305
1306     def test_POST_upload_mutable(self):
1307         # this creates a mutable file
1308         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1309                       file=("new.txt", self.NEWFILE_CONTENTS))
1310         fn = self._foo_node
1311         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1312         d.addCallback(lambda res:
1313                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1314                                                              self.NEWFILE_CONTENTS))
1315         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1316         def _got(newnode):
1317             self.failUnless(IMutableFileNode.providedBy(newnode))
1318             self.failUnless(newnode.is_mutable())
1319             self.failIf(newnode.is_readonly())
1320             self._mutable_node = newnode
1321             self._mutable_uri = newnode.get_uri()
1322         d.addCallback(_got)
1323
1324         # now upload it again and make sure that the URI doesn't change
1325         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1326         d.addCallback(lambda res:
1327                       self.POST(self.public_url + "/foo", t="upload",
1328                                 mutable="true",
1329                                 file=("new.txt", NEWER_CONTENTS)))
1330         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1331         d.addCallback(lambda res:
1332                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1333                                                              NEWER_CONTENTS))
1334         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1335         def _got2(newnode):
1336             self.failUnless(IMutableFileNode.providedBy(newnode))
1337             self.failUnless(newnode.is_mutable())
1338             self.failIf(newnode.is_readonly())
1339             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1340         d.addCallback(_got2)
1341
1342         # upload a second time, using PUT instead of POST
1343         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1344         d.addCallback(lambda res:
1345                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1346         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1347         d.addCallback(lambda res:
1348                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1349                                                              NEW2_CONTENTS))
1350
1351         # finally list the directory, since mutable files are displayed
1352         # slightly differently
1353
1354         d.addCallback(lambda res:
1355                       self.GET(self.public_url + "/foo/",
1356                                followRedirect=True))
1357         def _check_page(res):
1358             # TODO: assert more about the contents
1359             self.failUnless("SSK" in res)
1360             return res
1361         d.addCallback(_check_page)
1362
1363         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1364         def _got3(newnode):
1365             self.failUnless(IMutableFileNode.providedBy(newnode))
1366             self.failUnless(newnode.is_mutable())
1367             self.failIf(newnode.is_readonly())
1368             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1369         d.addCallback(_got3)
1370
1371         # look at the JSON form of the enclosing directory
1372         d.addCallback(lambda res:
1373                       self.GET(self.public_url + "/foo/?t=json",
1374                                followRedirect=True))
1375         def _check_page_json(res):
1376             parsed = simplejson.loads(res)
1377             self.failUnlessEqual(parsed[0], "dirnode")
1378             children = dict( [(unicode(name),value)
1379                               for (name,value)
1380                               in parsed[1]["children"].iteritems()] )
1381             self.failUnless("new.txt" in children)
1382             new_json = children["new.txt"]
1383             self.failUnlessEqual(new_json[0], "filenode")
1384             self.failUnless(new_json[1]["mutable"])
1385             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1386             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1387             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1388         d.addCallback(_check_page_json)
1389
1390         # and the JSON form of the file
1391         d.addCallback(lambda res:
1392                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1393         def _check_file_json(res):
1394             parsed = simplejson.loads(res)
1395             self.failUnlessEqual(parsed[0], "filenode")
1396             self.failUnless(parsed[1]["mutable"])
1397             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1398             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1399             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1400         d.addCallback(_check_file_json)
1401
1402         # and look at t=uri and t=readonly-uri
1403         d.addCallback(lambda res:
1404                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1405         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1406         d.addCallback(lambda res:
1407                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1408         def _check_ro_uri(res):
1409             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1410             self.failUnlessEqual(res, ro_uri)
1411         d.addCallback(_check_ro_uri)
1412
1413         # make sure we can get to it from /uri/URI
1414         d.addCallback(lambda res:
1415                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1416         d.addCallback(lambda res:
1417                       self.failUnlessEqual(res, NEW2_CONTENTS))
1418
1419         # and that HEAD computes the size correctly
1420         d.addCallback(lambda res:
1421                       self.HEAD(self.public_url + "/foo/new.txt",
1422                                 return_response=True))
1423         def _got_headers((res, status, headers)):
1424             self.failUnlessEqual(res, "")
1425             self.failUnlessEqual(headers["content-length"][0],
1426                                  str(len(NEW2_CONTENTS)))
1427             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1428         d.addCallback(_got_headers)
1429
1430         # make sure that size errors are displayed correctly for overwrite
1431         d.addCallback(lambda res:
1432                       self.shouldFail2(error.Error,
1433                                        "test_POST_upload_mutable-toobig",
1434                                        "413 Request Entity Too Large",
1435                                        "SDMF is limited to one segment, and 10001 > 10000",
1436                                        self.POST,
1437                                        self.public_url + "/foo", t="upload",
1438                                        mutable="true",
1439                                        file=("new.txt",
1440                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1441                                        ))
1442
1443         d.addErrback(self.dump_error)
1444         return d
1445
1446     def test_POST_upload_mutable_toobig(self):
1447         d = self.shouldFail2(error.Error,
1448                              "test_POST_upload_no_link_mutable_toobig",
1449                              "413 Request Entity Too Large",
1450                              "SDMF is limited to one segment, and 10001 > 10000",
1451                              self.POST,
1452                              self.public_url + "/foo",
1453                              t="upload", mutable="true",
1454                              file=("new.txt",
1455                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1456         return d
1457
1458     def dump_error(self, f):
1459         # if the web server returns an error code (like 400 Bad Request),
1460         # web.client.getPage puts the HTTP response body into the .response
1461         # attribute of the exception object that it gives back. It does not
1462         # appear in the Failure's repr(), so the ERROR that trial displays
1463         # will be rather terse and unhelpful. addErrback this method to the
1464         # end of your chain to get more information out of these errors.
1465         if f.check(error.Error):
1466             print "web.error.Error:"
1467             print f
1468             print f.value.response
1469         return f
1470
1471     def test_POST_upload_replace(self):
1472         d = self.POST(self.public_url + "/foo", t="upload",
1473                       file=("bar.txt", self.NEWFILE_CONTENTS))
1474         fn = self._foo_node
1475         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1476         d.addCallback(lambda res:
1477                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1478                                                       self.NEWFILE_CONTENTS))
1479         return d
1480
1481     def test_POST_upload_no_replace_ok(self):
1482         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1483                       file=("new.txt", self.NEWFILE_CONTENTS))
1484         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1485         d.addCallback(lambda res: self.failUnlessEqual(res,
1486                                                        self.NEWFILE_CONTENTS))
1487         return d
1488
1489     def test_POST_upload_no_replace_queryarg(self):
1490         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1491                       file=("bar.txt", self.NEWFILE_CONTENTS))
1492         d.addBoth(self.shouldFail, error.Error,
1493                   "POST_upload_no_replace_queryarg",
1494                   "409 Conflict",
1495                   "There was already a child by that name, and you asked me "
1496                   "to not replace it")
1497         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1498         d.addCallback(self.failUnlessIsBarDotTxt)
1499         return d
1500
1501     def test_POST_upload_no_replace_field(self):
1502         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1503                       file=("bar.txt", self.NEWFILE_CONTENTS))
1504         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1505                   "409 Conflict",
1506                   "There was already a child by that name, and you asked me "
1507                   "to not replace it")
1508         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1509         d.addCallback(self.failUnlessIsBarDotTxt)
1510         return d
1511
1512     def test_POST_upload_whendone(self):
1513         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1514                       file=("new.txt", self.NEWFILE_CONTENTS))
1515         d.addBoth(self.shouldRedirect, "/THERE")
1516         fn = self._foo_node
1517         d.addCallback(lambda res:
1518                       self.failUnlessChildContentsAre(fn, u"new.txt",
1519                                                       self.NEWFILE_CONTENTS))
1520         return d
1521
1522     def test_POST_upload_named(self):
1523         fn = self._foo_node
1524         d = self.POST(self.public_url + "/foo", t="upload",
1525                       name="new.txt", file=self.NEWFILE_CONTENTS)
1526         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1527         d.addCallback(lambda res:
1528                       self.failUnlessChildContentsAre(fn, u"new.txt",
1529                                                       self.NEWFILE_CONTENTS))
1530         return d
1531
1532     def test_POST_upload_named_badfilename(self):
1533         d = self.POST(self.public_url + "/foo", t="upload",
1534                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1535         d.addBoth(self.shouldFail, error.Error,
1536                   "test_POST_upload_named_badfilename",
1537                   "400 Bad Request",
1538                   "name= may not contain a slash",
1539                   )
1540         # make sure that nothing was added
1541         d.addCallback(lambda res:
1542                       self.failUnlessNodeKeysAre(self._foo_node,
1543                                                  [u"bar.txt", u"blockingfile",
1544                                                   u"empty", u"n\u00fc.txt",
1545                                                   u"sub"]))
1546         return d
1547
1548     def test_POST_FILEURL_check(self):
1549         bar_url = self.public_url + "/foo/bar.txt"
1550         d = self.POST(bar_url, t="check")
1551         def _check(res):
1552             self.failUnless("Healthy :" in res)
1553         d.addCallback(_check)
1554         redir_url = "http://allmydata.org/TARGET"
1555         def _check2(statuscode, target):
1556             self.failUnlessEqual(statuscode, str(http.FOUND))
1557             self.failUnlessEqual(target, redir_url)
1558         d.addCallback(lambda res:
1559                       self.shouldRedirect2("test_POST_FILEURL_check",
1560                                            _check2,
1561                                            self.POST, bar_url,
1562                                            t="check",
1563                                            when_done=redir_url))
1564         d.addCallback(lambda res:
1565                       self.POST(bar_url, t="check", return_to=redir_url))
1566         def _check3(res):
1567             self.failUnless("Healthy :" in res)
1568             self.failUnless("Return to parent directory" in res)
1569             self.failUnless(redir_url in res)
1570         d.addCallback(_check3)
1571
1572         d.addCallback(lambda res:
1573                       self.POST(bar_url, t="check", output="JSON"))
1574         def _check_json(res):
1575             data = simplejson.loads(res)
1576             self.failUnless("storage-index" in data)
1577             self.failUnless(data["results"]["healthy"])
1578         d.addCallback(_check_json)
1579
1580         return d
1581
1582     def test_POST_FILEURL_check_and_repair(self):
1583         bar_url = self.public_url + "/foo/bar.txt"
1584         d = self.POST(bar_url, t="check", repair="true")
1585         def _check(res):
1586             self.failUnless("Healthy :" in res)
1587         d.addCallback(_check)
1588         redir_url = "http://allmydata.org/TARGET"
1589         def _check2(statuscode, target):
1590             self.failUnlessEqual(statuscode, str(http.FOUND))
1591             self.failUnlessEqual(target, redir_url)
1592         d.addCallback(lambda res:
1593                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1594                                            _check2,
1595                                            self.POST, bar_url,
1596                                            t="check", repair="true",
1597                                            when_done=redir_url))
1598         d.addCallback(lambda res:
1599                       self.POST(bar_url, t="check", return_to=redir_url))
1600         def _check3(res):
1601             self.failUnless("Healthy :" in res)
1602             self.failUnless("Return to parent directory" in res)
1603             self.failUnless(redir_url in res)
1604         d.addCallback(_check3)
1605         return d
1606
1607     def test_POST_DIRURL_check(self):
1608         foo_url = self.public_url + "/foo/"
1609         d = self.POST(foo_url, t="check")
1610         def _check(res):
1611             self.failUnless("Healthy :" in res, res)
1612         d.addCallback(_check)
1613         redir_url = "http://allmydata.org/TARGET"
1614         def _check2(statuscode, target):
1615             self.failUnlessEqual(statuscode, str(http.FOUND))
1616             self.failUnlessEqual(target, redir_url)
1617         d.addCallback(lambda res:
1618                       self.shouldRedirect2("test_POST_DIRURL_check",
1619                                            _check2,
1620                                            self.POST, foo_url,
1621                                            t="check",
1622                                            when_done=redir_url))
1623         d.addCallback(lambda res:
1624                       self.POST(foo_url, t="check", return_to=redir_url))
1625         def _check3(res):
1626             self.failUnless("Healthy :" in res, res)
1627             self.failUnless("Return to parent directory" in res)
1628             self.failUnless(redir_url in res)
1629         d.addCallback(_check3)
1630
1631         d.addCallback(lambda res:
1632                       self.POST(foo_url, t="check", output="JSON"))
1633         def _check_json(res):
1634             data = simplejson.loads(res)
1635             self.failUnless("storage-index" in data)
1636             self.failUnless(data["results"]["healthy"])
1637         d.addCallback(_check_json)
1638
1639         return d
1640
1641     def test_POST_DIRURL_check_and_repair(self):
1642         foo_url = self.public_url + "/foo/"
1643         d = self.POST(foo_url, t="check", repair="true")
1644         def _check(res):
1645             self.failUnless("Healthy :" in res, res)
1646         d.addCallback(_check)
1647         redir_url = "http://allmydata.org/TARGET"
1648         def _check2(statuscode, target):
1649             self.failUnlessEqual(statuscode, str(http.FOUND))
1650             self.failUnlessEqual(target, redir_url)
1651         d.addCallback(lambda res:
1652                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1653                                            _check2,
1654                                            self.POST, foo_url,
1655                                            t="check", repair="true",
1656                                            when_done=redir_url))
1657         d.addCallback(lambda res:
1658                       self.POST(foo_url, t="check", return_to=redir_url))
1659         def _check3(res):
1660             self.failUnless("Healthy :" in res)
1661             self.failUnless("Return to parent directory" in res)
1662             self.failUnless(redir_url in res)
1663         d.addCallback(_check3)
1664         return d
1665
1666     def wait_for_operation(self, ignored, ophandle):
1667         url = "/operations/" + ophandle
1668         url += "?t=status&output=JSON"
1669         d = self.GET(url)
1670         def _got(res):
1671             data = simplejson.loads(res)
1672             if not data["finished"]:
1673                 d = self.stall(delay=1.0)
1674                 d.addCallback(self.wait_for_operation, ophandle)
1675                 return d
1676             return data
1677         d.addCallback(_got)
1678         return d
1679
1680     def get_operation_results(self, ignored, ophandle, output=None):
1681         url = "/operations/" + ophandle
1682         url += "?t=status"
1683         if output:
1684             url += "&output=" + output
1685         d = self.GET(url)
1686         def _got(res):
1687             if output and output.lower() == "json":
1688                 return simplejson.loads(res)
1689             return res
1690         d.addCallback(_got)
1691         return d
1692
1693     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1694         d = self.shouldFail2(error.Error,
1695                              "test_POST_DIRURL_deepcheck_no_ophandle",
1696                              "400 Bad Request",
1697                              "slow operation requires ophandle=",
1698                              self.POST, self.public_url, t="start-deep-check")
1699         return d
1700
1701     def test_POST_DIRURL_deepcheck(self):
1702         def _check_redirect(statuscode, target):
1703             self.failUnlessEqual(statuscode, str(http.FOUND))
1704             self.failUnless(target.endswith("/operations/123"))
1705         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1706                                  self.POST, self.public_url,
1707                                  t="start-deep-check", ophandle="123")
1708         d.addCallback(self.wait_for_operation, "123")
1709         def _check_json(data):
1710             self.failUnlessEqual(data["finished"], True)
1711             self.failUnlessEqual(data["count-objects-checked"], 8)
1712             self.failUnlessEqual(data["count-objects-healthy"], 8)
1713         d.addCallback(_check_json)
1714         d.addCallback(self.get_operation_results, "123", "html")
1715         def _check_html(res):
1716             self.failUnless("Objects Checked: <span>8</span>" in res)
1717             self.failUnless("Objects Healthy: <span>8</span>" in res)
1718         d.addCallback(_check_html)
1719
1720         d.addCallback(lambda res:
1721                       self.GET("/operations/123/"))
1722         d.addCallback(_check_html) # should be the same as without the slash
1723
1724         d.addCallback(lambda res:
1725                       self.shouldFail2(error.Error, "one", "404 Not Found",
1726                                        "No detailed results for SI bogus",
1727                                        self.GET, "/operations/123/bogus"))
1728
1729         foo_si = self._foo_node.get_storage_index()
1730         foo_si_s = base32.b2a(foo_si)
1731         d.addCallback(lambda res:
1732                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1733         def _check_foo_json(res):
1734             data = simplejson.loads(res)
1735             self.failUnlessEqual(data["storage-index"], foo_si_s)
1736             self.failUnless(data["results"]["healthy"])
1737         d.addCallback(_check_foo_json)
1738         return d
1739
1740     def test_POST_DIRURL_deepcheck_and_repair(self):
1741         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1742                       ophandle="124", output="json", followRedirect=True)
1743         d.addCallback(self.wait_for_operation, "124")
1744         def _check_json(data):
1745             self.failUnlessEqual(data["finished"], True)
1746             self.failUnlessEqual(data["count-objects-checked"], 8)
1747             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1748             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1749             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1750             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1751             self.failUnlessEqual(data["count-repairs-successful"], 0)
1752             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1753             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1754             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1755             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1756         d.addCallback(_check_json)
1757         d.addCallback(self.get_operation_results, "124", "html")
1758         def _check_html(res):
1759             self.failUnless("Objects Checked: <span>8</span>" in res)
1760
1761             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1762             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1763             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1764
1765             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1766             self.failUnless("Repairs Successful: <span>0</span>" in res)
1767             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1768
1769             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1770             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1771             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1772         d.addCallback(_check_html)
1773         return d
1774
1775     def test_POST_FILEURL_bad_t(self):
1776         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1777                              "POST to file: bad t=bogus",
1778                              self.POST, self.public_url + "/foo/bar.txt",
1779                              t="bogus")
1780         return d
1781
1782     def test_POST_mkdir(self): # return value?
1783         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1784         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1785         d.addCallback(self.failUnlessNodeKeysAre, [])
1786         return d
1787
1788     def test_POST_mkdir_2(self):
1789         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1790         d.addCallback(lambda res:
1791                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1792         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1793         d.addCallback(self.failUnlessNodeKeysAre, [])
1794         return d
1795
1796     def test_POST_mkdirs_2(self):
1797         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1798         d.addCallback(lambda res:
1799                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1800         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1801         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1802         d.addCallback(self.failUnlessNodeKeysAre, [])
1803         return d
1804
1805     def test_POST_mkdir_no_parentdir_noredirect(self):
1806         d = self.POST("/uri?t=mkdir")
1807         def _after_mkdir(res):
1808             uri.NewDirectoryURI.init_from_string(res)
1809         d.addCallback(_after_mkdir)
1810         return d
1811
1812     def test_POST_mkdir_no_parentdir_redirect(self):
1813         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1814         d.addBoth(self.shouldRedirect, None, statuscode='303')
1815         def _check_target(target):
1816             target = urllib.unquote(target)
1817             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1818         d.addCallback(_check_target)
1819         return d
1820
1821     def test_POST_noparent_bad(self):
1822         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1823                                   "/uri accepts only PUT, PUT?t=mkdir, "
1824                                   "POST?t=upload, and POST?t=mkdir",
1825                                   self.POST, "/uri?t=bogus")
1826         return d
1827
1828     def test_welcome_page_mkdir_button(self):
1829         # Fetch the welcome page.
1830         d = self.GET("/")
1831         def _after_get_welcome_page(res):
1832             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1833             mo = MKDIR_BUTTON_RE.search(res)
1834             formaction = mo.group(1)
1835             formt = mo.group(2)
1836             formaname = mo.group(3)
1837             formavalue = mo.group(4)
1838             return (formaction, formt, formaname, formavalue)
1839         d.addCallback(_after_get_welcome_page)
1840         def _after_parse_form(res):
1841             (formaction, formt, formaname, formavalue) = res
1842             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1843         d.addCallback(_after_parse_form)
1844         d.addBoth(self.shouldRedirect, None, statuscode='303')
1845         return d
1846
1847     def test_POST_mkdir_replace(self): # return value?
1848         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1849         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1850         d.addCallback(self.failUnlessNodeKeysAre, [])
1851         return d
1852
1853     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1854         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1855         d.addBoth(self.shouldFail, error.Error,
1856                   "POST_mkdir_no_replace_queryarg",
1857                   "409 Conflict",
1858                   "There was already a child by that name, and you asked me "
1859                   "to not replace it")
1860         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1861         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1862         return d
1863
1864     def test_POST_mkdir_no_replace_field(self): # return value?
1865         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1866                       replace="false")
1867         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1868                   "409 Conflict",
1869                   "There was already a child by that name, and you asked me "
1870                   "to not replace it")
1871         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1872         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1873         return d
1874
1875     def test_POST_mkdir_whendone_field(self):
1876         d = self.POST(self.public_url + "/foo",
1877                       t="mkdir", name="newdir", when_done="/THERE")
1878         d.addBoth(self.shouldRedirect, "/THERE")
1879         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1880         d.addCallback(self.failUnlessNodeKeysAre, [])
1881         return d
1882
1883     def test_POST_mkdir_whendone_queryarg(self):
1884         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1885                       t="mkdir", name="newdir")
1886         d.addBoth(self.shouldRedirect, "/THERE")
1887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1888         d.addCallback(self.failUnlessNodeKeysAre, [])
1889         return d
1890
1891     def test_POST_bad_t(self):
1892         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1893                              "POST to a directory with bad t=BOGUS",
1894                              self.POST, self.public_url + "/foo", t="BOGUS")
1895         return d
1896
1897     def test_POST_set_children(self):
1898         contents9, n9, newuri9 = self.makefile(9)
1899         contents10, n10, newuri10 = self.makefile(10)
1900         contents11, n11, newuri11 = self.makefile(11)
1901
1902         reqbody = """{
1903                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1904                                                 "size": 0,
1905                                                 "metadata": {
1906                                                   "ctime": 1002777696.7564139,
1907                                                   "mtime": 1002777696.7564139
1908                                                  }
1909                                                } ],
1910                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1911                                                 "size": 1,
1912                                                 "metadata": {
1913                                                   "ctime": 1002777696.7564139,
1914                                                   "mtime": 1002777696.7564139
1915                                                  }
1916                                                } ],
1917                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1918                                                 "size": 2,
1919                                                 "metadata": {
1920                                                   "ctime": 1002777696.7564139,
1921                                                   "mtime": 1002777696.7564139
1922                                                  }
1923                                                } ]
1924                     }""" % (newuri9, newuri10, newuri11)
1925
1926         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1927
1928         d = client.getPage(url, method="POST", postdata=reqbody)
1929         def _then(res):
1930             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1931             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1932             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1933
1934         d.addCallback(_then)
1935         d.addErrback(self.dump_error)
1936         return d
1937
1938     def test_POST_put_uri(self):
1939         contents, n, newuri = self.makefile(8)
1940         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1941         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1942         d.addCallback(lambda res:
1943                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1944                                                       contents))
1945         return d
1946
1947     def test_POST_put_uri_replace(self):
1948         contents, n, newuri = self.makefile(8)
1949         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1950         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1951         d.addCallback(lambda res:
1952                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1953                                                       contents))
1954         return d
1955
1956     def test_POST_put_uri_no_replace_queryarg(self):
1957         contents, n, newuri = self.makefile(8)
1958         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1959                       name="bar.txt", uri=newuri)
1960         d.addBoth(self.shouldFail, error.Error,
1961                   "POST_put_uri_no_replace_queryarg",
1962                   "409 Conflict",
1963                   "There was already a child by that name, and you asked me "
1964                   "to not replace it")
1965         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1966         d.addCallback(self.failUnlessIsBarDotTxt)
1967         return d
1968
1969     def test_POST_put_uri_no_replace_field(self):
1970         contents, n, newuri = self.makefile(8)
1971         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1972                       name="bar.txt", uri=newuri)
1973         d.addBoth(self.shouldFail, error.Error,
1974                   "POST_put_uri_no_replace_field",
1975                   "409 Conflict",
1976                   "There was already a child by that name, and you asked me "
1977                   "to not replace it")
1978         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1979         d.addCallback(self.failUnlessIsBarDotTxt)
1980         return d
1981
1982     def test_POST_delete(self):
1983         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1984         d.addCallback(lambda res: self._foo_node.list())
1985         def _check(children):
1986             self.failIf(u"bar.txt" in children)
1987         d.addCallback(_check)
1988         return d
1989
1990     def test_POST_rename_file(self):
1991         d = self.POST(self.public_url + "/foo", t="rename",
1992                       from_name="bar.txt", to_name='wibble.txt')
1993         d.addCallback(lambda res:
1994                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1995         d.addCallback(lambda res:
1996                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1997         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1998         d.addCallback(self.failUnlessIsBarDotTxt)
1999         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2000         d.addCallback(self.failUnlessIsBarJSON)
2001         return d
2002
2003     def test_POST_rename_file_redundant(self):
2004         d = self.POST(self.public_url + "/foo", t="rename",
2005                       from_name="bar.txt", to_name='bar.txt')
2006         d.addCallback(lambda res:
2007                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2008         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2009         d.addCallback(self.failUnlessIsBarDotTxt)
2010         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2011         d.addCallback(self.failUnlessIsBarJSON)
2012         return d
2013
2014     def test_POST_rename_file_replace(self):
2015         # rename a file and replace a directory with it
2016         d = self.POST(self.public_url + "/foo", t="rename",
2017                       from_name="bar.txt", to_name='empty')
2018         d.addCallback(lambda res:
2019                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2020         d.addCallback(lambda res:
2021                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2022         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2023         d.addCallback(self.failUnlessIsBarDotTxt)
2024         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2025         d.addCallback(self.failUnlessIsBarJSON)
2026         return d
2027
2028     def test_POST_rename_file_no_replace_queryarg(self):
2029         # rename a file and replace a directory with it
2030         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2031                       from_name="bar.txt", to_name='empty')
2032         d.addBoth(self.shouldFail, error.Error,
2033                   "POST_rename_file_no_replace_queryarg",
2034                   "409 Conflict",
2035                   "There was already a child by that name, and you asked me "
2036                   "to not replace it")
2037         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2038         d.addCallback(self.failUnlessIsEmptyJSON)
2039         return d
2040
2041     def test_POST_rename_file_no_replace_field(self):
2042         # rename a file and replace a directory with it
2043         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2044                       from_name="bar.txt", to_name='empty')
2045         d.addBoth(self.shouldFail, error.Error,
2046                   "POST_rename_file_no_replace_field",
2047                   "409 Conflict",
2048                   "There was already a child by that name, and you asked me "
2049                   "to not replace it")
2050         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2051         d.addCallback(self.failUnlessIsEmptyJSON)
2052         return d
2053
2054     def failUnlessIsEmptyJSON(self, res):
2055         data = simplejson.loads(res)
2056         self.failUnlessEqual(data[0], "dirnode", data)
2057         self.failUnlessEqual(len(data[1]["children"]), 0)
2058
2059     def test_POST_rename_file_slash_fail(self):
2060         d = self.POST(self.public_url + "/foo", t="rename",
2061                       from_name="bar.txt", to_name='kirk/spock.txt')
2062         d.addBoth(self.shouldFail, error.Error,
2063                   "test_POST_rename_file_slash_fail",
2064                   "400 Bad Request",
2065                   "to_name= may not contain a slash",
2066                   )
2067         d.addCallback(lambda res:
2068                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2069         return d
2070
2071     def test_POST_rename_dir(self):
2072         d = self.POST(self.public_url, t="rename",
2073                       from_name="foo", to_name='plunk')
2074         d.addCallback(lambda res:
2075                       self.failIfNodeHasChild(self.public_root, u"foo"))
2076         d.addCallback(lambda res:
2077                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2078         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2079         d.addCallback(self.failUnlessIsFooJSON)
2080         return d
2081
2082     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2083         """ If target is not None then the redirection has to go to target.  If
2084         statuscode is not None then the redirection has to be accomplished with
2085         that HTTP status code."""
2086         if not isinstance(res, failure.Failure):
2087             to_where = (target is None) and "somewhere" or ("to " + target)
2088             self.fail("%s: we were expecting to get redirected %s, not get an"
2089                       " actual page: %s" % (which, to_where, res))
2090         res.trap(error.PageRedirect)
2091         if statuscode is not None:
2092             self.failUnlessEqual(res.value.status, statuscode,
2093                                  "%s: not a redirect" % which)
2094         if target is not None:
2095             # the PageRedirect does not seem to capture the uri= query arg
2096             # properly, so we can't check for it.
2097             realtarget = self.webish_url + target
2098             self.failUnlessEqual(res.value.location, realtarget,
2099                                  "%s: wrong target" % which)
2100         return res.value.location
2101
2102     def test_GET_URI_form(self):
2103         base = "/uri?uri=%s" % self._bar_txt_uri
2104         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2105         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2106         d = self.GET(base)
2107         d.addBoth(self.shouldRedirect, targetbase)
2108         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2109         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2110         d.addCallback(lambda res: self.GET(base+"&t=json"))
2111         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2112         d.addCallback(self.log, "about to get file by uri")
2113         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2114         d.addCallback(self.failUnlessIsBarDotTxt)
2115         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2116         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2117                                            followRedirect=True))
2118         d.addCallback(self.failUnlessIsFooJSON)
2119         d.addCallback(self.log, "got dir by uri")
2120
2121         return d
2122
2123     def test_GET_URI_form_bad(self):
2124         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2125                              "400 Bad Request", "GET /uri requires uri=",
2126                              self.GET, "/uri")
2127         return d
2128
2129     def test_GET_rename_form(self):
2130         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2131                      followRedirect=True)
2132         def _check(res):
2133             self.failUnless('name="when_done" value="."' in res, res)
2134             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2135         d.addCallback(_check)
2136         return d
2137
2138     def log(self, res, msg):
2139         #print "MSG: %s  RES: %s" % (msg, res)
2140         log.msg(msg)
2141         return res
2142
2143     def test_GET_URI_URL(self):
2144         base = "/uri/%s" % self._bar_txt_uri
2145         d = self.GET(base)
2146         d.addCallback(self.failUnlessIsBarDotTxt)
2147         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2148         d.addCallback(self.failUnlessIsBarDotTxt)
2149         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2150         d.addCallback(self.failUnlessIsBarDotTxt)
2151         return d
2152
2153     def test_GET_URI_URL_dir(self):
2154         base = "/uri/%s?t=json" % self._foo_uri
2155         d = self.GET(base)
2156         d.addCallback(self.failUnlessIsFooJSON)
2157         return d
2158
2159     def test_GET_URI_URL_missing(self):
2160         base = "/uri/%s" % self._bad_file_uri
2161         d = self.GET(base)
2162         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2163                   http.GONE, response_substring="NotEnoughSharesError")
2164         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2165         # here? we must arrange for a download to fail after target.open()
2166         # has been called, and then inspect the response to see that it is
2167         # shorter than we expected.
2168         return d
2169
2170     def test_PUT_DIRURL_uri(self):
2171         d = self.s.create_empty_dirnode()
2172         def _made_dir(dn):
2173             new_uri = dn.get_uri()
2174             # replace /foo with a new (empty) directory
2175             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2176             d.addCallback(lambda res:
2177                           self.failUnlessEqual(res.strip(), new_uri))
2178             d.addCallback(lambda res:
2179                           self.failUnlessChildURIIs(self.public_root,
2180                                                     u"foo",
2181                                                     new_uri))
2182             return d
2183         d.addCallback(_made_dir)
2184         return d
2185
2186     def test_PUT_DIRURL_uri_noreplace(self):
2187         d = self.s.create_empty_dirnode()
2188         def _made_dir(dn):
2189             new_uri = dn.get_uri()
2190             # replace /foo with a new (empty) directory, but ask that
2191             # replace=false, so it should fail
2192             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2193                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2194                                  self.PUT,
2195                                  self.public_url + "/foo?t=uri&replace=false",
2196                                  new_uri)
2197             d.addCallback(lambda res:
2198                           self.failUnlessChildURIIs(self.public_root,
2199                                                     u"foo",
2200                                                     self._foo_uri))
2201             return d
2202         d.addCallback(_made_dir)
2203         return d
2204
2205     def test_PUT_DIRURL_bad_t(self):
2206         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2207                                  "400 Bad Request", "PUT to a directory",
2208                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2209         d.addCallback(lambda res:
2210                       self.failUnlessChildURIIs(self.public_root,
2211                                                 u"foo",
2212                                                 self._foo_uri))
2213         return d
2214
2215     def test_PUT_NEWFILEURL_uri(self):
2216         contents, n, new_uri = self.makefile(8)
2217         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2218         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2219         d.addCallback(lambda res:
2220                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2221                                                       contents))
2222         return d
2223
2224     def test_PUT_NEWFILEURL_uri_replace(self):
2225         contents, n, new_uri = self.makefile(8)
2226         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2227         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2228         d.addCallback(lambda res:
2229                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2230                                                       contents))
2231         return d
2232
2233     def test_PUT_NEWFILEURL_uri_no_replace(self):
2234         contents, n, new_uri = self.makefile(8)
2235         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2236         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2237                   "409 Conflict",
2238                   "There was already a child by that name, and you asked me "
2239                   "to not replace it")
2240         return d
2241
2242     def test_PUT_NEWFILE_URI(self):
2243         file_contents = "New file contents here\n"
2244         d = self.PUT("/uri", file_contents)
2245         def _check(uri):
2246             self.failUnless(uri in FakeCHKFileNode.all_contents)
2247             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2248                                  file_contents)
2249             return self.GET("/uri/%s" % uri)
2250         d.addCallback(_check)
2251         def _check2(res):
2252             self.failUnlessEqual(res, file_contents)
2253         d.addCallback(_check2)
2254         return d
2255
2256     def test_PUT_NEWFILE_URI_only_PUT(self):
2257         d = self.PUT("/uri?t=bogus", "")
2258         d.addBoth(self.shouldFail, error.Error,
2259                   "PUT_NEWFILE_URI_only_PUT",
2260                   "400 Bad Request",
2261                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2262         return d
2263
2264     def test_PUT_NEWFILE_URI_mutable(self):
2265         file_contents = "New file contents here\n"
2266         d = self.PUT("/uri?mutable=true", file_contents)
2267         def _check_mutable(uri):
2268             uri = uri.strip()
2269             u = IURI(uri)
2270             self.failUnless(IMutableFileURI.providedBy(u))
2271             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2272             n = self.s.create_node_from_uri(uri)
2273             return n.download_best_version()
2274         d.addCallback(_check_mutable)
2275         def _check2_mutable(data):
2276             self.failUnlessEqual(data, file_contents)
2277         d.addCallback(_check2_mutable)
2278         return d
2279
2280         def _check(uri):
2281             self.failUnless(uri in FakeCHKFileNode.all_contents)
2282             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2283                                  file_contents)
2284             return self.GET("/uri/%s" % uri)
2285         d.addCallback(_check)
2286         def _check2(res):
2287             self.failUnlessEqual(res, file_contents)
2288         d.addCallback(_check2)
2289         return d
2290
2291     def test_PUT_mkdir(self):
2292         d = self.PUT("/uri?t=mkdir", "")
2293         def _check(uri):
2294             n = self.s.create_node_from_uri(uri.strip())
2295             d2 = self.failUnlessNodeKeysAre(n, [])
2296             d2.addCallback(lambda res:
2297                            self.GET("/uri/%s?t=json" % uri))
2298             return d2
2299         d.addCallback(_check)
2300         d.addCallback(self.failUnlessIsEmptyJSON)
2301         return d
2302
2303     def test_POST_check(self):
2304         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2305         def _done(res):
2306             # this returns a string form of the results, which are probably
2307             # None since we're using fake filenodes.
2308             # TODO: verify that the check actually happened, by changing
2309             # FakeCHKFileNode to count how many times .check() has been
2310             # called.
2311             pass
2312         d.addCallback(_done)
2313         return d
2314
2315     def test_bad_method(self):
2316         url = self.webish_url + self.public_url + "/foo/bar.txt"
2317         d = self.shouldHTTPError2("test_bad_method",
2318                                   501, "Not Implemented",
2319                                   "I don't know how to treat a BOGUS request.",
2320                                   client.getPage, url, method="BOGUS")
2321         return d
2322
2323     def test_short_url(self):
2324         url = self.webish_url + "/uri"
2325         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2326                                   "I don't know how to treat a DELETE request.",
2327                                   client.getPage, url, method="DELETE")
2328         return d
2329
2330     def test_ophandle_bad(self):
2331         url = self.webish_url + "/operations/bogus?t=status"
2332         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2333                                   "unknown/expired handle 'bogus'",
2334                                   client.getPage, url)
2335         return d
2336
2337     def test_ophandle_cancel(self):
2338         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2339                       followRedirect=True)
2340         d.addCallback(lambda ignored:
2341                       self.GET("/operations/128?t=status&output=JSON"))
2342         def _check1(res):
2343             data = simplejson.loads(res)
2344             self.failUnless("finished" in data, res)
2345             monitor = self.ws.root.child_operations.handles["128"][0]
2346             d = self.POST("/operations/128?t=cancel&output=JSON")
2347             def _check2(res):
2348                 data = simplejson.loads(res)
2349                 self.failUnless("finished" in data, res)
2350                 # t=cancel causes the handle to be forgotten
2351                 self.failUnless(monitor.is_cancelled())
2352             d.addCallback(_check2)
2353             return d
2354         d.addCallback(_check1)
2355         d.addCallback(lambda ignored:
2356                       self.shouldHTTPError2("test_ophandle_cancel",
2357                                             404, "404 Not Found",
2358                                             "unknown/expired handle '128'",
2359                                             self.GET,
2360                                             "/operations/128?t=status&output=JSON"))
2361         return d
2362
2363     def test_ophandle_retainfor(self):
2364         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2365                       followRedirect=True)
2366         d.addCallback(lambda ignored:
2367                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2368         def _check1(res):
2369             data = simplejson.loads(res)
2370             self.failUnless("finished" in data, res)
2371         d.addCallback(_check1)
2372         # the retain-for=0 will cause the handle to be expired very soon
2373         d.addCallback(self.stall, 2.0)
2374         d.addCallback(lambda ignored:
2375                       self.shouldHTTPError2("test_ophandle_retainfor",
2376                                             404, "404 Not Found",
2377                                             "unknown/expired handle '129'",
2378                                             self.GET,
2379                                             "/operations/129?t=status&output=JSON"))
2380         return d
2381
2382     def test_ophandle_release_after_complete(self):
2383         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2384                       followRedirect=True)
2385         d.addCallback(self.wait_for_operation, "130")
2386         d.addCallback(lambda ignored:
2387                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2388         # the release-after-complete=true will cause the handle to be expired
2389         d.addCallback(lambda ignored:
2390                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2391                                             404, "404 Not Found",
2392                                             "unknown/expired handle '130'",
2393                                             self.GET,
2394                                             "/operations/130?t=status&output=JSON"))
2395         return d
2396
2397     def test_incident(self):
2398         d = self.POST("/report_incident", details="eek")
2399         def _done(res):
2400             self.failUnless("Thank you for your report!" in res, res)
2401         d.addCallback(_done)
2402         return d
2403
2404     def test_static(self):
2405         webdir = os.path.join(self.staticdir, "subdir")
2406         fileutil.make_dirs(webdir)
2407         f = open(os.path.join(webdir, "hello.txt"), "wb")
2408         f.write("hello")
2409         f.close()
2410
2411         d = self.GET("/static/subdir/hello.txt")
2412         def _check(res):
2413             self.failUnlessEqual(res, "hello")
2414         d.addCallback(_check)
2415         return d
2416
2417
2418 class Util(unittest.TestCase):
2419     def test_abbreviate_time(self):
2420         self.failUnlessEqual(common.abbreviate_time(None), "")
2421         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2422         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2423         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2424         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2425
2426     def test_abbreviate_rate(self):
2427         self.failUnlessEqual(common.abbreviate_rate(None), "")
2428         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2429         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2430         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2431
2432     def test_abbreviate_size(self):
2433         self.failUnlessEqual(common.abbreviate_size(None), "")
2434         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2435         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2436         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2437         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2438