src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17 import common_util as testutil
  18
  19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  20 # create a webserver that works against them
  21
  22 class FakeIntroducerClient:
  23     def get_all_connectors(self):
  24         return {}
  25     def get_all_connections_for(self, service_name):
  26         return frozenset()
  27     def get_all_peerids(self):
  28         return frozenset()
  29
  30 class FakeClient(service.MultiService):
  31     nodeid = "fake_nodeid"
  32     nickname = "fake_nickname"
  33     basedir = "fake_basedir"
  34     def get_versions(self):
  35         return {'allmydata': "fake",
  36                 'foolscap': "fake",
  37                 'twisted': "fake",
  38                 'zfec': "fake",
  39                 }
  40     introducer_furl = "None"
  41     introducer_client = FakeIntroducerClient()
  42     _all_upload_status = [upload.UploadStatus()]
  43     _all_download_status = [download.DownloadStatus()]
  44     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  45     _all_publish_statuses = [publish.PublishStatus()]
  46     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  47     convergence = "some random string"
  48
  49     def connected_to_introducer(self):
  50         return False
  51
  52     def get_nickname_for_peerid(self, peerid):
  53         return u"John Doe"
  54
  55     def create_node_from_uri(self, auri):
  56         u = uri.from_string(auri)
  57         if (INewDirectoryURI.providedBy(u)
  58             or IReadonlyNewDirectoryURI.providedBy(u)):
  59             return FakeDirectoryNode(self).init_from_uri(u)
  60         if IFileURI.providedBy(u):
  61             return FakeCHKFileNode(u, self)
  62         assert IMutableFileURI.providedBy(u), u
  63         return FakeMutableFileNode(self).init_from_uri(u)
  64
  65     def create_empty_dirnode(self):
  66         n = FakeDirectoryNode(self)
  67         d = n.create()
  68         d.addCallback(lambda res: n)
  69         return d
  70
  71     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  72     def create_mutable_file(self, contents=""):
  73         n = FakeMutableFileNode(self)
  74         return n.create(contents)
  75
  76     def upload(self, uploadable):
  77         d = uploadable.get_size()
  78         d.addCallback(lambda size: uploadable.read(size))
  79         def _got_data(datav):
  80             data = "".join(datav)
  81             n = create_chk_filenode(self, data)
  82             results = upload.UploadResults()
  83             results.uri = n.get_uri()
  84             return results
  85         d.addCallback(_got_data)
  86         return d
  87
  88     def list_all_upload_statuses(self):
  89         return self._all_upload_status
  90     def list_all_download_statuses(self):
  91         return self._all_download_status
  92     def list_all_mapupdate_statuses(self):
  93         return self._all_mapupdate_statuses
  94     def list_all_publish_statuses(self):
  95         return self._all_publish_statuses
  96     def list_all_retrieve_statuses(self):
  97         return self._all_retrieve_statuses
  98     def list_all_helper_statuses(self):
  99         return []
 100
 101 class MyGetter(client.HTTPPageGetter):
 102     handleStatus_206 = lambda self: self.handleStatus_200()
 103
 104 class HTTPClientHEADFactory(client.HTTPClientFactory):
 105     protocol = MyGetter
 106
 107     def noPage(self, reason):
 108         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 109         # exception when the response to a HEAD request had no body (when in
 110         # fact they are defined to never have a body). This was fixed in
 111         # Twisted-8.0 . To work around this, we catch the
 112         # PartialDownloadError and make it disappear.
 113         if (reason.check(client.PartialDownloadError)
 114             and self.method.upper() == "HEAD"):
 115             self.page("")
 116             return
 117         return client.HTTPClientFactory.noPage(self, reason)
 118
 119 class HTTPClientGETFactory(client.HTTPClientFactory):
 120     protocol = MyGetter
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 128         s.setServiceParent(self.s)
 129         self.webish_port = port = s.listener._port.getHost().port
 130         self.webish_url = "http://localhost:%d" % port
 131
 132         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 133         d = defer.DeferredList(l)
 134         def _then(res):
 135             self.public_root = res[0][1]
 136             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 137             self.public_url = "/uri/" + self.public_root.get_uri()
 138             self.private_root = res[1][1]
 139
 140             foo = res[2][1]
 141             self._foo_node = foo
 142             self._foo_uri = foo.get_uri()
 143             self._foo_readonly_uri = foo.get_readonly_uri()
 144             # NOTE: we ignore the deferred on all set_uri() calls, because we
 145             # know the fake nodes do these synchronously
 146             self.public_root.set_uri(u"foo", foo.get_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 150
 151             foo.set_uri(u"empty", res[3][1].get_uri())
 152             sub_uri = res[4][1].get_uri()
 153             self._sub_uri = sub_uri
 154             foo.set_uri(u"sub", sub_uri)
 155             sub = self.s.create_node_from_uri(sub_uri)
 156
 157             _ign, n, blocking_uri = self.makefile(1)
 158             foo.set_uri(u"blockingfile", blocking_uri)
 159
 160             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 161             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 162             # still think of it as an umlaut
 163             foo.set_uri(unicode_filename, self._bar_txt_uri)
 164
 165             _ign, n, baz_file = self.makefile(2)
 166             sub.set_uri(u"baz.txt", baz_file)
 167
 168             _ign, n, self._bad_file_uri = self.makefile(3)
 169             # this uri should not be downloadable
 170             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 171
 172             rodir = res[5][1]
 173             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(self.s, contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 214
 215     def failUnlessIsFooJSON(self, res):
 216         data = simplejson.loads(res)
 217         self.failUnless(isinstance(data, list))
 218         self.failUnlessEqual(data[0], "dirnode", res)
 219         self.failUnless(isinstance(data[1], dict))
 220         self.failUnless(data[1]["mutable"])
 221         self.failUnless("rw_uri" in data[1]) # mutable
 222         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 223         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 224
 225         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 226         self.failUnlessEqual(kidnames,
 227                              [u"bar.txt", u"blockingfile", u"empty",
 228                               u"n\u00fc.txt", u"sub"])
 229         kids = dict( [(unicode(name),value)
 230                       for (name,value)
 231                       in data[1]["children"].iteritems()] )
 232         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 233         self.failUnless("metadata" in kids[u"sub"][1])
 234         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 235         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 236         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 237         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 238         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 240                              self._bar_txt_metadata["ctime"])
 241         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 242                              self._bar_txt_uri)
 243
 244     def GET(self, urlpath, followRedirect=False, return_response=False,
 245             **kwargs):
 246         # if return_response=True, this fires with (data, statuscode,
 247         # respheaders) instead of just data.
 248         assert not isinstance(urlpath, unicode)
 249         url = self.webish_url + urlpath
 250         factory = HTTPClientGETFactory(url, method="GET",
 251                                        followRedirect=followRedirect, **kwargs)
 252         reactor.connectTCP("localhost", self.webish_port, factory)
 253         d = factory.deferred
 254         def _got_data(data):
 255             return (data, factory.status, factory.response_headers)
 256         if return_response:
 257             d.addCallback(_got_data)
 258         return factory.deferred
 259
 260     def HEAD(self, urlpath, return_response=False, **kwargs):
 261         # this requires some surgery, because twisted.web.client doesn't want
 262         # to give us back the response headers.
 263         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 264         reactor.connectTCP("localhost", self.webish_port, factory)
 265         d = factory.deferred
 266         def _got_data(data):
 267             return (data, factory.status, factory.response_headers)
 268         if return_response:
 269             d.addCallback(_got_data)
 270         return factory.deferred
 271
 272     def PUT(self, urlpath, data, **kwargs):
 273         url = self.webish_url + urlpath
 274         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 275
 276     def DELETE(self, urlpath):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="DELETE")
 279
 280     def POST(self, urlpath, followRedirect=False, **fields):
 281         url = self.webish_url + urlpath
 282         sepbase = "boogabooga"
 283         sep = "--" + sepbase
 284         form = []
 285         form.append(sep)
 286         form.append('Content-Disposition: form-data; name="_charset"')
 287         form.append('')
 288         form.append('UTF-8')
 289         form.append(sep)
 290         for name, value in fields.iteritems():
 291             if isinstance(value, tuple):
 292                 filename, value = value
 293                 form.append('Content-Disposition: form-data; name="%s"; '
 294                             'filename="%s"' % (name, filename.encode("utf-8")))
 295             else:
 296                 form.append('Content-Disposition: form-data; name="%s"' % name)
 297             form.append('')
 298             if isinstance(value, unicode):
 299                 value = value.encode("utf-8")
 300             else:
 301                 value = str(value)
 302             assert isinstance(value, str)
 303             form.append(value)
 304             form.append(sep)
 305         form[-1] += "--"
 306         body = "\r\n".join(form) + "\r\n"
 307         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 308                    }
 309         return client.getPage(url, method="POST", postdata=body,
 310                               headers=headers, followRedirect=followRedirect)
 311
 312     def shouldFail(self, res, expected_failure, which,
 313                    substring=None, response_substring=None):
 314         if isinstance(res, failure.Failure):
 315             res.trap(expected_failure)
 316             if substring:
 317                 self.failUnless(substring in str(res),
 318                                 "substring '%s' not in '%s'"
 319                                 % (substring, str(res)))
 320             if response_substring:
 321                 self.failUnless(response_substring in res.value.response,
 322                                 "response substring '%s' not in '%s'"
 323                                 % (response_substring, res.value.response))
 324         else:
 325             self.fail("%s was supposed to raise %s, not get '%s'" %
 326                       (which, expected_failure, res))
 327
 328     def shouldFail2(self, expected_failure, which, substring,
 329                     response_substring,
 330                     callable, *args, **kwargs):
 331         assert substring is None or isinstance(substring, str)
 332         assert response_substring is None or isinstance(response_substring, str)
 333         d = defer.maybeDeferred(callable, *args, **kwargs)
 334         def done(res):
 335             if isinstance(res, failure.Failure):
 336                 res.trap(expected_failure)
 337                 if substring:
 338                     self.failUnless(substring in str(res),
 339                                     "%s: substring '%s' not in '%s'"
 340                                     % (which, substring, str(res)))
 341                 if response_substring:
 342                     self.failUnless(response_substring in res.value.response,
 343                                     "%s: response substring '%s' not in '%s'"
 344                                     % (which,
 345                                        response_substring, res.value.response))
 346             else:
 347                 self.fail("%s was supposed to raise %s, not get '%s'" %
 348                           (which, expected_failure, res))
 349         d.addBoth(done)
 350         return d
 351
 352     def should404(self, res, which):
 353         if isinstance(res, failure.Failure):
 354             res.trap(error.Error)
 355             self.failUnlessEqual(res.value.status, "404")
 356         else:
 357             self.fail("%s was supposed to Error(404), not get '%s'" %
 358                       (which, res))
 359
 360     def shouldHTTPError(self, res, which, code=None, substring=None,
 361                         response_substring=None):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             if code is not None:
 365                 self.failUnlessEqual(res.value.status, str(code))
 366             if substring:
 367                 self.failUnless(substring in str(res),
 368                                 "substring '%s' not in '%s'"
 369                                 % (substring, str(res)))
 370             if response_substring:
 371                 self.failUnless(response_substring in res.value.response,
 372                                 "response substring '%s' not in '%s'"
 373                                 % (response_substring, res.value.response))
 374         else:
 375             self.fail("%s was supposed to Error(%s), not get '%s'" %
 376                       (which, code, res))
 377
 378     def shouldHTTPError2(self, which,
 379                          code=None, substring=None, response_substring=None,
 380                          callable=None, *args, **kwargs):
 381         assert substring is None or isinstance(substring, str)
 382         assert callable
 383         d = defer.maybeDeferred(callable, *args, **kwargs)
 384         d.addBoth(self.shouldHTTPError, which,
 385                   code, substring, response_substring)
 386         return d
 387
 388
 389 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 390     def test_create(self):
 391         pass
 392
 393     def test_welcome(self):
 394         d = self.GET("/")
 395         def _check(res):
 396             self.failUnless('Welcome To AllMyData' in res)
 397             self.failUnless('Tahoe' in res)
 398
 399             self.s.basedir = 'web/test_welcome'
 400             fileutil.make_dirs("web/test_welcome")
 401             fileutil.make_dirs("web/test_welcome/private")
 402             return self.GET("/")
 403         d.addCallback(_check)
 404         return d
 405
 406     def test_provisioning_math(self):
 407         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 408         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 409         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 410         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 411         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 412
 413     def test_provisioning(self):
 414         d = self.GET("/provisioning/")
 415         def _check(res):
 416             self.failUnless('Tahoe Provisioning Tool' in res)
 417             fields = {'filled': True,
 418                       "num_users": int(50e3),
 419                       "files_per_user": 1000,
 420                       "space_per_user": int(1e9),
 421                       "sharing_ratio": 1.0,
 422                       "encoding_parameters": "3-of-10-5",
 423                       "num_servers": 30,
 424                       "ownership_mode": "A",
 425                       "download_rate": 100,
 426                       "upload_rate": 10,
 427                       "delete_rate": 10,
 428                       "lease_timer": 7,
 429                       }
 430             return self.POST("/provisioning/", **fields)
 431
 432         d.addCallback(_check)
 433         def _check2(res):
 434             self.failUnless('Tahoe Provisioning Tool' in res)
 435             self.failUnless("Share space consumed: 167.01TB" in res)
 436
 437             fields = {'filled': True,
 438                       "num_users": int(50e6),
 439                       "files_per_user": 1000,
 440                       "space_per_user": int(5e9),
 441                       "sharing_ratio": 1.0,
 442                       "encoding_parameters": "25-of-100-50",
 443                       "num_servers": 30000,
 444                       "ownership_mode": "E",
 445                       "drive_failure_model": "U",
 446                       "drive_size": 1000,
 447                       "download_rate": 1000,
 448                       "upload_rate": 100,
 449                       "delete_rate": 100,
 450                       "lease_timer": 7,
 451                       }
 452             return self.POST("/provisioning/", **fields)
 453         d.addCallback(_check2)
 454         def _check3(res):
 455             self.failUnless("Share space consumed: huge!" in res)
 456             fields = {'filled': True}
 457             return self.POST("/provisioning/", **fields)
 458         d.addCallback(_check3)
 459         def _check4(res):
 460             self.failUnless("Share space consumed:" in res)
 461         d.addCallback(_check4)
 462         return d
 463
 464     def test_status(self):
 465         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 466         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 467         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 468         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 469         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 470         d = self.GET("/status", followRedirect=True)
 471         def _check(res):
 472             self.failUnless('Upload and Download Status' in res, res)
 473             self.failUnless('"down-%d"' % dl_num in res, res)
 474             self.failUnless('"up-%d"' % ul_num in res, res)
 475             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 476             self.failUnless('"publish-%d"' % pub_num in res, res)
 477             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 478         d.addCallback(_check)
 479         d.addCallback(lambda res: self.GET("/status/?t=json"))
 480         def _check_json(res):
 481             data = simplejson.loads(res)
 482             self.failUnless(isinstance(data, dict))
 483             active = data["active"]
 484             # TODO: test more. We need a way to fake an active operation
 485             # here.
 486         d.addCallback(_check_json)
 487
 488         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 489         def _check_dl(res):
 490             self.failUnless("File Download Status" in res, res)
 491         d.addCallback(_check_dl)
 492         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 493         def _check_ul(res):
 494             self.failUnless("File Upload Status" in res, res)
 495         d.addCallback(_check_ul)
 496         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 497         def _check_mapupdate(res):
 498             self.failUnless("Mutable File Servermap Update Status" in res, res)
 499         d.addCallback(_check_mapupdate)
 500         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 501         def _check_publish(res):
 502             self.failUnless("Mutable File Publish Status" in res, res)
 503         d.addCallback(_check_publish)
 504         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 505         def _check_retrieve(res):
 506             self.failUnless("Mutable File Retrieve Status" in res, res)
 507         d.addCallback(_check_retrieve)
 508
 509         return d
 510
 511     def test_status_numbers(self):
 512         drrm = status.DownloadResultsRendererMixin()
 513         self.failUnlessEqual(drrm.render_time(None, None), "")
 514         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 515         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 517         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 518         self.failUnlessEqual(drrm.render_rate(None, None), "")
 519         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 521         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 522
 523         urrm = status.UploadResultsRendererMixin()
 524         self.failUnlessEqual(urrm.render_time(None, None), "")
 525         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 526         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 528         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 529         self.failUnlessEqual(urrm.render_rate(None, None), "")
 530         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 532         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 533
 534     def test_GET_FILEURL(self):
 535         d = self.GET(self.public_url + "/foo/bar.txt")
 536         d.addCallback(self.failUnlessIsBarDotTxt)
 537         return d
 538
 539     def test_GET_FILEURL_range(self):
 540         headers = {"range": "bytes=1-10"}
 541         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 542                      return_response=True)
 543         def _got((res, status, headers)):
 544             self.failUnlessEqual(int(status), 206)
 545             self.failUnless(headers.has_key("content-range"))
 546             self.failUnlessEqual(headers["content-range"][0],
 547                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 548             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 549         d.addCallback(_got)
 550         return d
 551
 552     def test_HEAD_FILEURL_range(self):
 553         headers = {"range": "bytes=1-10"}
 554         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 555                      return_response=True)
 556         def _got((res, status, headers)):
 557             self.failUnlessEqual(res, "")
 558             self.failUnlessEqual(int(status), 206)
 559             self.failUnless(headers.has_key("content-range"))
 560             self.failUnlessEqual(headers["content-range"][0],
 561                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 562         d.addCallback(_got)
 563         return d
 564
 565     def test_GET_FILEURL_range_bad(self):
 566         headers = {"range": "BOGUS=fizbop-quarnak"}
 567         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 568                              "400 Bad Request",
 569                              "Syntactically invalid http range header",
 570                              self.GET, self.public_url + "/foo/bar.txt",
 571                              headers=headers)
 572         return d
 573
 574     def test_HEAD_FILEURL(self):
 575         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 576         def _got((res, status, headers)):
 577             self.failUnlessEqual(res, "")
 578             self.failUnlessEqual(headers["content-length"][0],
 579                                  str(len(self.BAR_CONTENTS)))
 580             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 581         d.addCallback(_got)
 582         return d
 583
 584     def test_GET_FILEURL_named(self):
 585         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 586         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 587         d = self.GET(base + "/@@name=/blah.txt")
 588         d.addCallback(self.failUnlessIsBarDotTxt)
 589         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 590         d.addCallback(self.failUnlessIsBarDotTxt)
 591         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 592         d.addCallback(self.failUnlessIsBarDotTxt)
 593         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 594         d.addCallback(self.failUnlessIsBarDotTxt)
 595         save_url = base + "?save=true&filename=blah.txt"
 596         d.addCallback(lambda res: self.GET(save_url))
 597         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 598         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 599         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 600         u_url = base + "?save=true&filename=" + u_fn_e
 601         d.addCallback(lambda res: self.GET(u_url))
 602         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 603         return d
 604
 605     def test_PUT_FILEURL_named_bad(self):
 606         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 607         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 608                              "400 Bad Request",
 609                              "/file can only be used with GET or HEAD",
 610                              self.PUT, base + "/@@name=/blah.txt", "")
 611         return d
 612
 613     def test_GET_DIRURL_named_bad(self):
 614         base = "/file/%s" % urllib.quote(self._foo_uri)
 615         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 616                              "400 Bad Request",
 617                              "is not a file-cap",
 618                              self.GET, base + "/@@name=/blah.txt")
 619         return d
 620
 621     def test_GET_slash_file_bad(self):
 622         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 623                              "404 Not Found",
 624                              "/file must be followed by a file-cap and a name",
 625                              self.GET, "/file")
 626         return d
 627
 628     def test_GET_unhandled_URI_named(self):
 629         contents, n, newuri = self.makefile(12)
 630         verifier_cap = n.get_verifier().to_string()
 631         base = "/file/%s" % urllib.quote(verifier_cap)
 632         # client.create_node_from_uri() can't handle verify-caps
 633         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 634                              "400 Bad Request",
 635                              "is not a valid file- or directory- cap",
 636                              self.GET, base)
 637         return d
 638
 639     def test_GET_unhandled_URI(self):
 640         contents, n, newuri = self.makefile(12)
 641         verifier_cap = n.get_verifier().to_string()
 642         base = "/uri/%s" % urllib.quote(verifier_cap)
 643         # client.create_node_from_uri() can't handle verify-caps
 644         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 645                              "400 Bad Request",
 646                              "is not a valid file- or directory- cap",
 647                              self.GET, base)
 648         return d
 649
 650     def test_GET_FILE_URI(self):
 651         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 652         d = self.GET(base)
 653         d.addCallback(self.failUnlessIsBarDotTxt)
 654         return d
 655
 656     def test_GET_FILE_URI_badchild(self):
 657         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 658         errmsg = "Files have no children, certainly not named 'boguschild'"
 659         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 660                              "400 Bad Request", errmsg,
 661                              self.GET, base)
 662         return d
 663
 664     def test_PUT_FILE_URI_badchild(self):
 665         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 666         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 667         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 668                              "400 Bad Request", errmsg,
 669                              self.PUT, base, "")
 670         return d
 671
 672     def test_GET_FILEURL_save(self):
 673         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 674         # TODO: look at the headers, expect a Content-Disposition: attachment
 675         # header.
 676         d.addCallback(self.failUnlessIsBarDotTxt)
 677         return d
 678
 679     def test_GET_FILEURL_missing(self):
 680         d = self.GET(self.public_url + "/foo/missing")
 681         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 682         return d
 683
 684     def test_PUT_NEWFILEURL(self):
 685         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 686         # TODO: we lose the response code, so we can't check this
 687         #self.failUnlessEqual(responsecode, 201)
 688         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 689         d.addCallback(lambda res:
 690                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 691                                                       self.NEWFILE_CONTENTS))
 692         return d
 693
 694     def test_PUT_NEWFILEURL_range_bad(self):
 695         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 696         target = self.public_url + "/foo/new.txt"
 697         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 698                              "501 Not Implemented",
 699                              "Content-Range in PUT not yet supported",
 700                              # (and certainly not for immutable files)
 701                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 702                              headers=headers)
 703         d.addCallback(lambda res:
 704                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 705         return d
 706
 707     def test_PUT_NEWFILEURL_mutable(self):
 708         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 709                      self.NEWFILE_CONTENTS)
 710         # TODO: we lose the response code, so we can't check this
 711         #self.failUnlessEqual(responsecode, 201)
 712         def _check_uri(res):
 713             u = uri.from_string_mutable_filenode(res)
 714             self.failUnless(u.is_mutable())
 715             self.failIf(u.is_readonly())
 716             return res
 717         d.addCallback(_check_uri)
 718         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 719         d.addCallback(lambda res:
 720                       self.failUnlessMutableChildContentsAre(self._foo_node,
 721                                                              u"new.txt",
 722                                                              self.NEWFILE_CONTENTS))
 723         return d
 724
 725     def test_PUT_NEWFILEURL_mutable_toobig(self):
 726         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 727                              "413 Request Entity Too Large",
 728                              "SDMF is limited to one segment, and 10001 > 10000",
 729                              self.PUT,
 730                              self.public_url + "/foo/new.txt?mutable=true",
 731                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 732         return d
 733
 734     def test_PUT_NEWFILEURL_replace(self):
 735         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 200)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_bad_t(self):
 745         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 746                              "PUT to a file: bad t=bogus",
 747                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 748                              "contents")
 749         return d
 750
 751     def test_PUT_NEWFILEURL_no_replace(self):
 752         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 753                      self.NEWFILE_CONTENTS)
 754         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 755                   "409 Conflict",
 756                   "There was already a child by that name, and you asked me "
 757                   "to not replace it")
 758         return d
 759
 760     def test_PUT_NEWFILEURL_mkdirs(self):
 761         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 762         fn = self._foo_node
 763         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 764         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 765         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_blocked(self):
 772         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 773                      self.NEWFILE_CONTENTS)
 774         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 775                   "409 Conflict",
 776                   "Unable to create directory 'blockingfile': a file was in the way")
 777         return d
 778
 779     def test_DELETE_FILEURL(self):
 780         d = self.DELETE(self.public_url + "/foo/bar.txt")
 781         d.addCallback(lambda res:
 782                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 783         return d
 784
 785     def test_DELETE_FILEURL_missing(self):
 786         d = self.DELETE(self.public_url + "/foo/missing")
 787         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 788         return d
 789
 790     def test_DELETE_FILEURL_missing2(self):
 791         d = self.DELETE(self.public_url + "/missing/missing")
 792         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 793         return d
 794
 795     def test_GET_FILEURL_json(self):
 796         # twisted.web.http.parse_qs ignores any query args without an '=', so
 797         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 798         # instead. This may make it tricky to emulate the S3 interface
 799         # completely.
 800         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 801         d.addCallback(self.failUnlessIsBarJSON)
 802         return d
 803
 804     def test_GET_FILEURL_json_missing(self):
 805         d = self.GET(self.public_url + "/foo/missing?json")
 806         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 807         return d
 808
 809     def test_GET_FILEURL_uri(self):
 810         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 811         def _check(res):
 812             self.failUnlessEqual(res, self._bar_txt_uri)
 813         d.addCallback(_check)
 814         d.addCallback(lambda res:
 815                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 816         def _check2(res):
 817             # for now, for files, uris and readonly-uris are the same
 818             self.failUnlessEqual(res, self._bar_txt_uri)
 819         d.addCallback(_check2)
 820         return d
 821
 822     def test_GET_FILEURL_badtype(self):
 823         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 824                                   "bad t=bogus",
 825                                   self.GET,
 826                                   self.public_url + "/foo/bar.txt?t=bogus")
 827         return d
 828
 829     def test_GET_FILEURL_uri_missing(self):
 830         d = self.GET(self.public_url + "/foo/missing?t=uri")
 831         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 832         return d
 833
 834     def test_GET_DIRURL(self):
 835         # the addSlash means we get a redirect here
 836         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 837         ROOT = "../../.."
 838         d = self.GET(self.public_url + "/foo", followRedirect=True)
 839         def _check(res):
 840             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 841                             in res, res)
 842             # the FILE reference points to a URI, but it should end in bar.txt
 843             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 844                        (ROOT, urllib.quote(self._bar_txt_uri)))
 845             get_bar = "".join([r'<td>',
 846                                r'<a href="%s">bar.txt</a>' % bar_url,
 847                                r'</td>',
 848                                r'\s+<td>FILE</td>',
 849                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 850                                ])
 851             self.failUnless(re.search(get_bar, res), res)
 852             for line in res.split("\n"):
 853                 # find the line that contains the delete button for bar.txt
 854                 if ("form action" in line and
 855                     'value="delete"' in line and
 856                     'value="bar.txt"' in line):
 857                     # the form target should use a relative URL
 858                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 859                     self.failUnless(('action="%s"' % foo_url) in line, line)
 860                     # and the when_done= should too
 861                     #done_url = urllib.quote(???)
 862                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 863                     #                in line, line)
 864                     break
 865             else:
 866                 self.fail("unable to find delete-bar.txt line", res)
 867
 868             # the DIR reference just points to a URI
 869             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 870             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 871                        + r'\s+<td>DIR</td>')
 872             self.failUnless(re.search(get_sub, res), res)
 873         d.addCallback(_check)
 874
 875         # look at a directory which is readonly
 876         d.addCallback(lambda res:
 877                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 878         def _check2(res):
 879             self.failUnless("(readonly)" in res, res)
 880             self.failIf("Upload a file" in res, res)
 881         d.addCallback(_check2)
 882
 883         # and at a directory that contains a readonly directory
 884         d.addCallback(lambda res:
 885                       self.GET(self.public_url, followRedirect=True))
 886         def _check3(res):
 887             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 888                                       '</td>\s+<td>DIR-RO</td>', res))
 889         d.addCallback(_check3)
 890
 891         return d
 892
 893     def test_GET_DIRURL_badtype(self):
 894         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 895                                   400, "Bad Request",
 896                                   "bad t=bogus",
 897                                   self.GET,
 898                                   self.public_url + "/foo?t=bogus")
 899         return d
 900
 901     def test_GET_DIRURL_json(self):
 902         d = self.GET(self.public_url + "/foo?t=json")
 903         d.addCallback(self.failUnlessIsFooJSON)
 904         return d
 905
 906
 907     def test_POST_DIRURL_manifest_no_ophandle(self):
 908         d = self.shouldFail2(error.Error,
 909                              "test_POST_DIRURL_manifest_no_ophandle",
 910                              "400 Bad Request",
 911                              "slow operation requires ophandle=",
 912                              self.POST, self.public_url, t="start-manifest")
 913         return d
 914
 915     def test_POST_DIRURL_manifest(self):
 916         d = defer.succeed(None)
 917         def getman(ignored, output):
 918             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 919                           followRedirect=True)
 920             d.addCallback(self.wait_for_operation, "125")
 921             d.addCallback(self.get_operation_results, "125", output)
 922             return d
 923         d.addCallback(getman, None)
 924         def _got_html(manifest):
 925             self.failUnless("Manifest of SI=" in manifest)
 926             self.failUnless("<td>sub</td>" in manifest)
 927             self.failUnless(self._sub_uri in manifest)
 928             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 929         d.addCallback(_got_html)
 930
 931         # both t=status and unadorned GET should be identical
 932         d.addCallback(lambda res: self.GET("/operations/125"))
 933         d.addCallback(_got_html)
 934
 935         d.addCallback(getman, "html")
 936         d.addCallback(_got_html)
 937         d.addCallback(getman, "text")
 938         def _got_text(manifest):
 939             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 940             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 941         d.addCallback(_got_text)
 942         d.addCallback(getman, "JSON")
 943         def _got_json(manifest):
 944             data = manifest["manifest"]
 945             got = {}
 946             for (path_list, cap) in data:
 947                 got[tuple(path_list)] = cap
 948             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 949             self.failUnless((u"sub",u"baz.txt") in got)
 950         d.addCallback(_got_json)
 951         return d
 952
 953     def test_POST_DIRURL_deepsize_no_ophandle(self):
 954         d = self.shouldFail2(error.Error,
 955                              "test_POST_DIRURL_deepsize_no_ophandle",
 956                              "400 Bad Request",
 957                              "slow operation requires ophandle=",
 958                              self.POST, self.public_url, t="start-deep-size")
 959         return d
 960
 961     def test_POST_DIRURL_deepsize(self):
 962         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 963                       followRedirect=True)
 964         d.addCallback(self.wait_for_operation, "126")
 965         d.addCallback(self.get_operation_results, "126", "json")
 966         def _got_json(data):
 967             self.failUnlessEqual(data["finished"], True)
 968             size = data["size"]
 969             self.failUnless(size > 1000)
 970         d.addCallback(_got_json)
 971         d.addCallback(self.get_operation_results, "126", "text")
 972         def _got_text(res):
 973             mo = re.search(r'^size: (\d+)$', res, re.M)
 974             self.failUnless(mo, res)
 975             size = int(mo.group(1))
 976             # with directories, the size varies.
 977             self.failUnless(size > 1000)
 978         d.addCallback(_got_text)
 979         return d
 980
 981     def test_POST_DIRURL_deepstats_no_ophandle(self):
 982         d = self.shouldFail2(error.Error,
 983                              "test_POST_DIRURL_deepstats_no_ophandle",
 984                              "400 Bad Request",
 985                              "slow operation requires ophandle=",
 986                              self.POST, self.public_url, t="start-deep-stats")
 987         return d
 988
 989     def test_POST_DIRURL_deepstats(self):
 990         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 991                       followRedirect=True)
 992         d.addCallback(self.wait_for_operation, "127")
 993         d.addCallback(self.get_operation_results, "127", "json")
 994         def _got_json(stats):
 995             expected = {"count-immutable-files": 3,
 996                         "count-mutable-files": 0,
 997                         "count-literal-files": 0,
 998                         "count-files": 3,
 999                         "count-directories": 3,
1000                         "size-immutable-files": 57,
1001                         "size-literal-files": 0,
1002                         #"size-directories": 1912, # varies
1003                         #"largest-directory": 1590,
1004                         "largest-directory-children": 5,
1005                         "largest-immutable-file": 19,
1006                         }
1007             for k,v in expected.iteritems():
1008                 self.failUnlessEqual(stats[k], v,
1009                                      "stats[%s] was %s, not %s" %
1010                                      (k, stats[k], v))
1011             self.failUnlessEqual(stats["size-files-histogram"],
1012                                  [ [11, 31, 3] ])
1013         d.addCallback(_got_json)
1014         return d
1015
1016     def test_GET_DIRURL_uri(self):
1017         d = self.GET(self.public_url + "/foo?t=uri")
1018         def _check(res):
1019             self.failUnlessEqual(res, self._foo_uri)
1020         d.addCallback(_check)
1021         return d
1022
1023     def test_GET_DIRURL_readonly_uri(self):
1024         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1025         def _check(res):
1026             self.failUnlessEqual(res, self._foo_readonly_uri)
1027         d.addCallback(_check)
1028         return d
1029
1030     def test_PUT_NEWDIRURL(self):
1031         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1032         d.addCallback(lambda res:
1033                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1034         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1035         d.addCallback(self.failUnlessNodeKeysAre, [])
1036         return d
1037
1038     def test_PUT_NEWDIRURL_exists(self):
1039         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1040         d.addCallback(lambda res:
1041                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1042         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1043         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1044         return d
1045
1046     def test_PUT_NEWDIRURL_blocked(self):
1047         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1048                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1049                              self.PUT,
1050                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1051         d.addCallback(lambda res:
1052                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1053         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1054         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1055         return d
1056
1057     def test_PUT_NEWDIRURL_mkdir_p(self):
1058         d = defer.succeed(None)
1059         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1060         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1061         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1062         def mkdir_p(mkpnode):
1063             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1064             d = self.POST(url)
1065             def made_subsub(ssuri):
1066                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1067                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1068                 d = self.POST(url)
1069                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1070                 return d
1071             d.addCallback(made_subsub)
1072             return d
1073         d.addCallback(mkdir_p)
1074         return d
1075
1076     def test_PUT_NEWDIRURL_mkdirs(self):
1077         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1078         d.addCallback(lambda res:
1079                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1080         d.addCallback(lambda res:
1081                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1082         d.addCallback(lambda res:
1083                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1084         d.addCallback(self.failUnlessNodeKeysAre, [])
1085         return d
1086
1087     def test_DELETE_DIRURL(self):
1088         d = self.DELETE(self.public_url + "/foo")
1089         d.addCallback(lambda res:
1090                       self.failIfNodeHasChild(self.public_root, u"foo"))
1091         return d
1092
1093     def test_DELETE_DIRURL_missing(self):
1094         d = self.DELETE(self.public_url + "/foo/missing")
1095         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1096         d.addCallback(lambda res:
1097                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1098         return d
1099
1100     def test_DELETE_DIRURL_missing2(self):
1101         d = self.DELETE(self.public_url + "/missing")
1102         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1103         return d
1104
1105     def dump_root(self):
1106         print "NODEWALK"
1107         w = webish.DirnodeWalkerMixin()
1108         def visitor(childpath, childnode, metadata):
1109             print childpath
1110         d = w.walk(self.public_root, visitor)
1111         return d
1112
1113     def failUnlessNodeKeysAre(self, node, expected_keys):
1114         for k in expected_keys:
1115             assert isinstance(k, unicode)
1116         d = node.list()
1117         def _check(children):
1118             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1119         d.addCallback(_check)
1120         return d
1121     def failUnlessNodeHasChild(self, node, name):
1122         assert isinstance(name, unicode)
1123         d = node.list()
1124         def _check(children):
1125             self.failUnless(name in children)
1126         d.addCallback(_check)
1127         return d
1128     def failIfNodeHasChild(self, node, name):
1129         assert isinstance(name, unicode)
1130         d = node.list()
1131         def _check(children):
1132             self.failIf(name in children)
1133         d.addCallback(_check)
1134         return d
1135
1136     def failUnlessChildContentsAre(self, node, name, expected_contents):
1137         assert isinstance(name, unicode)
1138         d = node.get_child_at_path(name)
1139         d.addCallback(lambda node: node.download_to_data())
1140         def _check(contents):
1141             self.failUnlessEqual(contents, expected_contents)
1142         d.addCallback(_check)
1143         return d
1144
1145     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1146         assert isinstance(name, unicode)
1147         d = node.get_child_at_path(name)
1148         d.addCallback(lambda node: node.download_best_version())
1149         def _check(contents):
1150             self.failUnlessEqual(contents, expected_contents)
1151         d.addCallback(_check)
1152         return d
1153
1154     def failUnlessChildURIIs(self, node, name, expected_uri):
1155         assert isinstance(name, unicode)
1156         d = node.get_child_at_path(name)
1157         def _check(child):
1158             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1159         d.addCallback(_check)
1160         return d
1161
1162     def failUnlessURIMatchesChild(self, got_uri, node, name):
1163         assert isinstance(name, unicode)
1164         d = node.get_child_at_path(name)
1165         def _check(child):
1166             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1167         d.addCallback(_check)
1168         return d
1169
1170     def failUnlessCHKURIHasContents(self, got_uri, contents):
1171         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1172
1173     def test_POST_upload(self):
1174         d = self.POST(self.public_url + "/foo", t="upload",
1175                       file=("new.txt", self.NEWFILE_CONTENTS))
1176         fn = self._foo_node
1177         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1178         d.addCallback(lambda res:
1179                       self.failUnlessChildContentsAre(fn, u"new.txt",
1180                                                       self.NEWFILE_CONTENTS))
1181         return d
1182
1183     def test_POST_upload_unicode(self):
1184         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1185         d = self.POST(self.public_url + "/foo", t="upload",
1186                       file=(filename, self.NEWFILE_CONTENTS))
1187         fn = self._foo_node
1188         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1189         d.addCallback(lambda res:
1190                       self.failUnlessChildContentsAre(fn, filename,
1191                                                       self.NEWFILE_CONTENTS))
1192         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1193         d.addCallback(lambda res: self.GET(target_url))
1194         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1195                                                             self.NEWFILE_CONTENTS,
1196                                                             contents))
1197         return d
1198
1199     def test_POST_upload_unicode_named(self):
1200         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1201         d = self.POST(self.public_url + "/foo", t="upload",
1202                       name=filename,
1203                       file=("overridden", self.NEWFILE_CONTENTS))
1204         fn = self._foo_node
1205         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1206         d.addCallback(lambda res:
1207                       self.failUnlessChildContentsAre(fn, filename,
1208                                                       self.NEWFILE_CONTENTS))
1209         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1210         d.addCallback(lambda res: self.GET(target_url))
1211         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1212                                                             self.NEWFILE_CONTENTS,
1213                                                             contents))
1214         return d
1215
1216     def test_POST_upload_no_link(self):
1217         d = self.POST("/uri", t="upload",
1218                       file=("new.txt", self.NEWFILE_CONTENTS))
1219         def _check_upload_results(page):
1220             # this should be a page which describes the results of the upload
1221             # that just finished.
1222             self.failUnless("Upload Results:" in page)
1223             self.failUnless("URI:" in page)
1224             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1225             mo = uri_re.search(page)
1226             self.failUnless(mo, page)
1227             new_uri = mo.group(1)
1228             return new_uri
1229         d.addCallback(_check_upload_results)
1230         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1231         return d
1232
1233     def test_POST_upload_no_link_whendone(self):
1234         d = self.POST("/uri", t="upload", when_done="/",
1235                       file=("new.txt", self.NEWFILE_CONTENTS))
1236         d.addBoth(self.shouldRedirect, "/")
1237         return d
1238
1239     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1240         d = defer.maybeDeferred(callable, *args, **kwargs)
1241         def done(res):
1242             if isinstance(res, failure.Failure):
1243                 res.trap(error.PageRedirect)
1244                 statuscode = res.value.status
1245                 target = res.value.location
1246                 return checker(statuscode, target)
1247             self.fail("%s: callable was supposed to redirect, not return '%s'"
1248                       % (which, res))
1249         d.addBoth(done)
1250         return d
1251
1252     def test_POST_upload_no_link_whendone_results(self):
1253         def check(statuscode, target):
1254             self.failUnlessEqual(statuscode, str(http.FOUND))
1255             self.failUnless(target.startswith(self.webish_url), target)
1256             return client.getPage(target, method="GET")
1257         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1258                                  check,
1259                                  self.POST, "/uri", t="upload",
1260                                  when_done="/uri/%(uri)s",
1261                                  file=("new.txt", self.NEWFILE_CONTENTS))
1262         d.addCallback(lambda res:
1263                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1264         return d
1265
1266     def test_POST_upload_no_link_mutable(self):
1267         d = self.POST("/uri", t="upload", mutable="true",
1268                       file=("new.txt", self.NEWFILE_CONTENTS))
1269         def _check(new_uri):
1270             new_uri = new_uri.strip()
1271             self.new_uri = new_uri
1272             u = IURI(new_uri)
1273             self.failUnless(IMutableFileURI.providedBy(u))
1274             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1275             n = self.s.create_node_from_uri(new_uri)
1276             return n.download_best_version()
1277         d.addCallback(_check)
1278         def _check2(data):
1279             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1280             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1281         d.addCallback(_check2)
1282         def _check3(data):
1283             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1284             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1285         d.addCallback(_check3)
1286         def _check4(data):
1287             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1288         d.addCallback(_check4)
1289         return d
1290
1291     def test_POST_upload_no_link_mutable_toobig(self):
1292         d = self.shouldFail2(error.Error,
1293                              "test_POST_upload_no_link_mutable_toobig",
1294                              "413 Request Entity Too Large",
1295                              "SDMF is limited to one segment, and 10001 > 10000",
1296                              self.POST,
1297                              "/uri", t="upload", mutable="true",
1298                              file=("new.txt",
1299                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1300         return d
1301
1302     def test_POST_upload_mutable(self):
1303         # this creates a mutable file
1304         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1305                       file=("new.txt", self.NEWFILE_CONTENTS))
1306         fn = self._foo_node
1307         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1308         d.addCallback(lambda res:
1309                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1310                                                              self.NEWFILE_CONTENTS))
1311         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1312         def _got(newnode):
1313             self.failUnless(IMutableFileNode.providedBy(newnode))
1314             self.failUnless(newnode.is_mutable())
1315             self.failIf(newnode.is_readonly())
1316             self._mutable_node = newnode
1317             self._mutable_uri = newnode.get_uri()
1318         d.addCallback(_got)
1319
1320         # now upload it again and make sure that the URI doesn't change
1321         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1322         d.addCallback(lambda res:
1323                       self.POST(self.public_url + "/foo", t="upload",
1324                                 mutable="true",
1325                                 file=("new.txt", NEWER_CONTENTS)))
1326         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1327         d.addCallback(lambda res:
1328                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1329                                                              NEWER_CONTENTS))
1330         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1331         def _got2(newnode):
1332             self.failUnless(IMutableFileNode.providedBy(newnode))
1333             self.failUnless(newnode.is_mutable())
1334             self.failIf(newnode.is_readonly())
1335             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1336         d.addCallback(_got2)
1337
1338         # upload a second time, using PUT instead of POST
1339         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1340         d.addCallback(lambda res:
1341                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1342         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1343         d.addCallback(lambda res:
1344                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1345                                                              NEW2_CONTENTS))
1346
1347         # finally list the directory, since mutable files are displayed
1348         # slightly differently
1349
1350         d.addCallback(lambda res:
1351                       self.GET(self.public_url + "/foo/",
1352                                followRedirect=True))
1353         def _check_page(res):
1354             # TODO: assert more about the contents
1355             self.failUnless("SSK" in res)
1356             return res
1357         d.addCallback(_check_page)
1358
1359         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1360         def _got3(newnode):
1361             self.failUnless(IMutableFileNode.providedBy(newnode))
1362             self.failUnless(newnode.is_mutable())
1363             self.failIf(newnode.is_readonly())
1364             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1365         d.addCallback(_got3)
1366
1367         # look at the JSON form of the enclosing directory
1368         d.addCallback(lambda res:
1369                       self.GET(self.public_url + "/foo/?t=json",
1370                                followRedirect=True))
1371         def _check_page_json(res):
1372             parsed = simplejson.loads(res)
1373             self.failUnlessEqual(parsed[0], "dirnode")
1374             children = dict( [(unicode(name),value)
1375                               for (name,value)
1376                               in parsed[1]["children"].iteritems()] )
1377             self.failUnless("new.txt" in children)
1378             new_json = children["new.txt"]
1379             self.failUnlessEqual(new_json[0], "filenode")
1380             self.failUnless(new_json[1]["mutable"])
1381             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1382             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1383             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1384         d.addCallback(_check_page_json)
1385
1386         # and the JSON form of the file
1387         d.addCallback(lambda res:
1388                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1389         def _check_file_json(res):
1390             parsed = simplejson.loads(res)
1391             self.failUnlessEqual(parsed[0], "filenode")
1392             self.failUnless(parsed[1]["mutable"])
1393             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1394             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1395             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1396         d.addCallback(_check_file_json)
1397
1398         # and look at t=uri and t=readonly-uri
1399         d.addCallback(lambda res:
1400                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1401         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1402         d.addCallback(lambda res:
1403                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1404         def _check_ro_uri(res):
1405             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1406             self.failUnlessEqual(res, ro_uri)
1407         d.addCallback(_check_ro_uri)
1408
1409         # make sure we can get to it from /uri/URI
1410         d.addCallback(lambda res:
1411                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1412         d.addCallback(lambda res:
1413                       self.failUnlessEqual(res, NEW2_CONTENTS))
1414
1415         # and that HEAD computes the size correctly
1416         d.addCallback(lambda res:
1417                       self.HEAD(self.public_url + "/foo/new.txt",
1418                                 return_response=True))
1419         def _got_headers((res, status, headers)):
1420             self.failUnlessEqual(res, "")
1421             self.failUnlessEqual(headers["content-length"][0],
1422                                  str(len(NEW2_CONTENTS)))
1423             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1424         d.addCallback(_got_headers)
1425
1426         # make sure that size errors are displayed correctly for overwrite
1427         d.addCallback(lambda res:
1428                       self.shouldFail2(error.Error,
1429                                        "test_POST_upload_mutable-toobig",
1430                                        "413 Request Entity Too Large",
1431                                        "SDMF is limited to one segment, and 10001 > 10000",
1432                                        self.POST,
1433                                        self.public_url + "/foo", t="upload",
1434                                        mutable="true",
1435                                        file=("new.txt",
1436                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1437                                        ))
1438
1439         d.addErrback(self.dump_error)
1440         return d
1441
1442     def test_POST_upload_mutable_toobig(self):
1443         d = self.shouldFail2(error.Error,
1444                              "test_POST_upload_no_link_mutable_toobig",
1445                              "413 Request Entity Too Large",
1446                              "SDMF is limited to one segment, and 10001 > 10000",
1447                              self.POST,
1448                              self.public_url + "/foo",
1449                              t="upload", mutable="true",
1450                              file=("new.txt",
1451                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1452         return d
1453
1454     def dump_error(self, f):
1455         # if the web server returns an error code (like 400 Bad Request),
1456         # web.client.getPage puts the HTTP response body into the .response
1457         # attribute of the exception object that it gives back. It does not
1458         # appear in the Failure's repr(), so the ERROR that trial displays
1459         # will be rather terse and unhelpful. addErrback this method to the
1460         # end of your chain to get more information out of these errors.
1461         if f.check(error.Error):
1462             print "web.error.Error:"
1463             print f
1464             print f.value.response
1465         return f
1466
1467     def test_POST_upload_replace(self):
1468         d = self.POST(self.public_url + "/foo", t="upload",
1469                       file=("bar.txt", self.NEWFILE_CONTENTS))
1470         fn = self._foo_node
1471         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1472         d.addCallback(lambda res:
1473                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1474                                                       self.NEWFILE_CONTENTS))
1475         return d
1476
1477     def test_POST_upload_no_replace_ok(self):
1478         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1479                       file=("new.txt", self.NEWFILE_CONTENTS))
1480         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1481         d.addCallback(lambda res: self.failUnlessEqual(res,
1482                                                        self.NEWFILE_CONTENTS))
1483         return d
1484
1485     def test_POST_upload_no_replace_queryarg(self):
1486         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1487                       file=("bar.txt", self.NEWFILE_CONTENTS))
1488         d.addBoth(self.shouldFail, error.Error,
1489                   "POST_upload_no_replace_queryarg",
1490                   "409 Conflict",
1491                   "There was already a child by that name, and you asked me "
1492                   "to not replace it")
1493         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1494         d.addCallback(self.failUnlessIsBarDotTxt)
1495         return d
1496
1497     def test_POST_upload_no_replace_field(self):
1498         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1499                       file=("bar.txt", self.NEWFILE_CONTENTS))
1500         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1501                   "409 Conflict",
1502                   "There was already a child by that name, and you asked me "
1503                   "to not replace it")
1504         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1505         d.addCallback(self.failUnlessIsBarDotTxt)
1506         return d
1507
1508     def test_POST_upload_whendone(self):
1509         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1510                       file=("new.txt", self.NEWFILE_CONTENTS))
1511         d.addBoth(self.shouldRedirect, "/THERE")
1512         fn = self._foo_node
1513         d.addCallback(lambda res:
1514                       self.failUnlessChildContentsAre(fn, u"new.txt",
1515                                                       self.NEWFILE_CONTENTS))
1516         return d
1517
1518     def test_POST_upload_named(self):
1519         fn = self._foo_node
1520         d = self.POST(self.public_url + "/foo", t="upload",
1521                       name="new.txt", file=self.NEWFILE_CONTENTS)
1522         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1523         d.addCallback(lambda res:
1524                       self.failUnlessChildContentsAre(fn, u"new.txt",
1525                                                       self.NEWFILE_CONTENTS))
1526         return d
1527
1528     def test_POST_upload_named_badfilename(self):
1529         d = self.POST(self.public_url + "/foo", t="upload",
1530                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1531         d.addBoth(self.shouldFail, error.Error,
1532                   "test_POST_upload_named_badfilename",
1533                   "400 Bad Request",
1534                   "name= may not contain a slash",
1535                   )
1536         # make sure that nothing was added
1537         d.addCallback(lambda res:
1538                       self.failUnlessNodeKeysAre(self._foo_node,
1539                                                  [u"bar.txt", u"blockingfile",
1540                                                   u"empty", u"n\u00fc.txt",
1541                                                   u"sub"]))
1542         return d
1543
1544     def test_POST_FILEURL_check(self):
1545         bar_url = self.public_url + "/foo/bar.txt"
1546         d = self.POST(bar_url, t="check")
1547         def _check(res):
1548             self.failUnless("Healthy :" in res)
1549         d.addCallback(_check)
1550         redir_url = "http://allmydata.org/TARGET"
1551         def _check2(statuscode, target):
1552             self.failUnlessEqual(statuscode, str(http.FOUND))
1553             self.failUnlessEqual(target, redir_url)
1554         d.addCallback(lambda res:
1555                       self.shouldRedirect2("test_POST_FILEURL_check",
1556                                            _check2,
1557                                            self.POST, bar_url,
1558                                            t="check",
1559                                            when_done=redir_url))
1560         d.addCallback(lambda res:
1561                       self.POST(bar_url, t="check", return_to=redir_url))
1562         def _check3(res):
1563             self.failUnless("Healthy :" in res)
1564             self.failUnless("Return to parent directory" in res)
1565             self.failUnless(redir_url in res)
1566         d.addCallback(_check3)
1567
1568         d.addCallback(lambda res:
1569                       self.POST(bar_url, t="check", output="JSON"))
1570         def _check_json(res):
1571             data = simplejson.loads(res)
1572             self.failUnless("storage-index" in data)
1573             self.failUnless(data["results"]["healthy"])
1574         d.addCallback(_check_json)
1575
1576         return d
1577
1578     def test_POST_FILEURL_check_and_repair(self):
1579         bar_url = self.public_url + "/foo/bar.txt"
1580         d = self.POST(bar_url, t="check", repair="true")
1581         def _check(res):
1582             self.failUnless("Healthy :" in res)
1583         d.addCallback(_check)
1584         redir_url = "http://allmydata.org/TARGET"
1585         def _check2(statuscode, target):
1586             self.failUnlessEqual(statuscode, str(http.FOUND))
1587             self.failUnlessEqual(target, redir_url)
1588         d.addCallback(lambda res:
1589                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1590                                            _check2,
1591                                            self.POST, bar_url,
1592                                            t="check", repair="true",
1593                                            when_done=redir_url))
1594         d.addCallback(lambda res:
1595                       self.POST(bar_url, t="check", return_to=redir_url))
1596         def _check3(res):
1597             self.failUnless("Healthy :" in res)
1598             self.failUnless("Return to parent directory" in res)
1599             self.failUnless(redir_url in res)
1600         d.addCallback(_check3)
1601         return d
1602
1603     def test_POST_DIRURL_check(self):
1604         foo_url = self.public_url + "/foo/"
1605         d = self.POST(foo_url, t="check")
1606         def _check(res):
1607             self.failUnless("Healthy :" in res, res)
1608         d.addCallback(_check)
1609         redir_url = "http://allmydata.org/TARGET"
1610         def _check2(statuscode, target):
1611             self.failUnlessEqual(statuscode, str(http.FOUND))
1612             self.failUnlessEqual(target, redir_url)
1613         d.addCallback(lambda res:
1614                       self.shouldRedirect2("test_POST_DIRURL_check",
1615                                            _check2,
1616                                            self.POST, foo_url,
1617                                            t="check",
1618                                            when_done=redir_url))
1619         d.addCallback(lambda res:
1620                       self.POST(foo_url, t="check", return_to=redir_url))
1621         def _check3(res):
1622             self.failUnless("Healthy :" in res, res)
1623             self.failUnless("Return to parent directory" in res)
1624             self.failUnless(redir_url in res)
1625         d.addCallback(_check3)
1626
1627         d.addCallback(lambda res:
1628                       self.POST(foo_url, t="check", output="JSON"))
1629         def _check_json(res):
1630             data = simplejson.loads(res)
1631             self.failUnless("storage-index" in data)
1632             self.failUnless(data["results"]["healthy"])
1633         d.addCallback(_check_json)
1634
1635         return d
1636
1637     def test_POST_DIRURL_check_and_repair(self):
1638         foo_url = self.public_url + "/foo/"
1639         d = self.POST(foo_url, t="check", repair="true")
1640         def _check(res):
1641             self.failUnless("Healthy :" in res, res)
1642         d.addCallback(_check)
1643         redir_url = "http://allmydata.org/TARGET"
1644         def _check2(statuscode, target):
1645             self.failUnlessEqual(statuscode, str(http.FOUND))
1646             self.failUnlessEqual(target, redir_url)
1647         d.addCallback(lambda res:
1648                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1649                                            _check2,
1650                                            self.POST, foo_url,
1651                                            t="check", repair="true",
1652                                            when_done=redir_url))
1653         d.addCallback(lambda res:
1654                       self.POST(foo_url, t="check", return_to=redir_url))
1655         def _check3(res):
1656             self.failUnless("Healthy :" in res)
1657             self.failUnless("Return to parent directory" in res)
1658             self.failUnless(redir_url in res)
1659         d.addCallback(_check3)
1660         return d
1661
1662     def wait_for_operation(self, ignored, ophandle):
1663         url = "/operations/" + ophandle
1664         url += "?t=status&output=JSON"
1665         d = self.GET(url)
1666         def _got(res):
1667             data = simplejson.loads(res)
1668             if not data["finished"]:
1669                 d = self.stall(delay=1.0)
1670                 d.addCallback(self.wait_for_operation, ophandle)
1671                 return d
1672             return data
1673         d.addCallback(_got)
1674         return d
1675
1676     def get_operation_results(self, ignored, ophandle, output=None):
1677         url = "/operations/" + ophandle
1678         url += "?t=status"
1679         if output:
1680             url += "&output=" + output
1681         d = self.GET(url)
1682         def _got(res):
1683             if output and output.lower() == "json":
1684                 return simplejson.loads(res)
1685             return res
1686         d.addCallback(_got)
1687         return d
1688
1689     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1690         d = self.shouldFail2(error.Error,
1691                              "test_POST_DIRURL_deepcheck_no_ophandle",
1692                              "400 Bad Request",
1693                              "slow operation requires ophandle=",
1694                              self.POST, self.public_url, t="start-deep-check")
1695         return d
1696
1697     def test_POST_DIRURL_deepcheck(self):
1698         def _check_redirect(statuscode, target):
1699             self.failUnlessEqual(statuscode, str(http.FOUND))
1700             self.failUnless(target.endswith("/operations/123"))
1701         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1702                                  self.POST, self.public_url,
1703                                  t="start-deep-check", ophandle="123")
1704         d.addCallback(self.wait_for_operation, "123")
1705         def _check_json(data):
1706             self.failUnlessEqual(data["finished"], True)
1707             self.failUnlessEqual(data["count-objects-checked"], 8)
1708             self.failUnlessEqual(data["count-objects-healthy"], 8)
1709         d.addCallback(_check_json)
1710         d.addCallback(self.get_operation_results, "123", "html")
1711         def _check_html(res):
1712             self.failUnless("Objects Checked: <span>8</span>" in res)
1713             self.failUnless("Objects Healthy: <span>8</span>" in res)
1714         d.addCallback(_check_html)
1715
1716         d.addCallback(lambda res:
1717                       self.GET("/operations/123/"))
1718         d.addCallback(_check_html) # should be the same as without the slash
1719
1720         d.addCallback(lambda res:
1721                       self.shouldFail2(error.Error, "one", "404 Not Found",
1722                                        "No detailed results for SI bogus",
1723                                        self.GET, "/operations/123/bogus"))
1724
1725         foo_si = self._foo_node.get_storage_index()
1726         foo_si_s = base32.b2a(foo_si)
1727         d.addCallback(lambda res:
1728                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1729         def _check_foo_json(res):
1730             data = simplejson.loads(res)
1731             self.failUnlessEqual(data["storage-index"], foo_si_s)
1732             self.failUnless(data["results"]["healthy"])
1733         d.addCallback(_check_foo_json)
1734         return d
1735
1736     def test_POST_DIRURL_deepcheck_and_repair(self):
1737         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1738                       ophandle="124", output="json", followRedirect=True)
1739         d.addCallback(self.wait_for_operation, "124")
1740         def _check_json(data):
1741             self.failUnlessEqual(data["finished"], True)
1742             self.failUnlessEqual(data["count-objects-checked"], 8)
1743             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1744             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1745             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1746             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1747             self.failUnlessEqual(data["count-repairs-successful"], 0)
1748             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1749             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1750             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1751             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1752         d.addCallback(_check_json)
1753         d.addCallback(self.get_operation_results, "124", "html")
1754         def _check_html(res):
1755             self.failUnless("Objects Checked: <span>8</span>" in res)
1756
1757             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1758             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1759             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1760
1761             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1762             self.failUnless("Repairs Successful: <span>0</span>" in res)
1763             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1764
1765             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1766             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1767             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1768         d.addCallback(_check_html)
1769         return d
1770
1771     def test_POST_FILEURL_bad_t(self):
1772         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1773                              "POST to file: bad t=bogus",
1774                              self.POST, self.public_url + "/foo/bar.txt",
1775                              t="bogus")
1776         return d
1777
1778     def test_POST_mkdir(self): # return value?
1779         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1780         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1781         d.addCallback(self.failUnlessNodeKeysAre, [])
1782         return d
1783
1784     def test_POST_mkdir_2(self):
1785         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1786         d.addCallback(lambda res:
1787                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1789         d.addCallback(self.failUnlessNodeKeysAre, [])
1790         return d
1791
1792     def test_POST_mkdirs_2(self):
1793         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1794         d.addCallback(lambda res:
1795                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1796         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1797         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1798         d.addCallback(self.failUnlessNodeKeysAre, [])
1799         return d
1800
1801     def test_POST_mkdir_no_parentdir_noredirect(self):
1802         d = self.POST("/uri?t=mkdir")
1803         def _after_mkdir(res):
1804             uri.NewDirectoryURI.init_from_string(res)
1805         d.addCallback(_after_mkdir)
1806         return d
1807
1808     def test_POST_mkdir_no_parentdir_redirect(self):
1809         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1810         d.addBoth(self.shouldRedirect, None, statuscode='303')
1811         def _check_target(target):
1812             target = urllib.unquote(target)
1813             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1814         d.addCallback(_check_target)
1815         return d
1816
1817     def test_POST_noparent_bad(self):
1818         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1819                                   "/uri accepts only PUT, PUT?t=mkdir, "
1820                                   "POST?t=upload, and POST?t=mkdir",
1821                                   self.POST, "/uri?t=bogus")
1822         return d
1823
1824     def test_welcome_page_mkdir_button(self):
1825         # Fetch the welcome page.
1826         d = self.GET("/")
1827         def _after_get_welcome_page(res):
1828             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1829             mo = MKDIR_BUTTON_RE.search(res)
1830             formaction = mo.group(1)
1831             formt = mo.group(2)
1832             formaname = mo.group(3)
1833             formavalue = mo.group(4)
1834             return (formaction, formt, formaname, formavalue)
1835         d.addCallback(_after_get_welcome_page)
1836         def _after_parse_form(res):
1837             (formaction, formt, formaname, formavalue) = res
1838             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1839         d.addCallback(_after_parse_form)
1840         d.addBoth(self.shouldRedirect, None, statuscode='303')
1841         return d
1842
1843     def test_POST_mkdir_replace(self): # return value?
1844         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1845         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1846         d.addCallback(self.failUnlessNodeKeysAre, [])
1847         return d
1848
1849     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1850         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1851         d.addBoth(self.shouldFail, error.Error,
1852                   "POST_mkdir_no_replace_queryarg",
1853                   "409 Conflict",
1854                   "There was already a child by that name, and you asked me "
1855                   "to not replace it")
1856         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1857         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1858         return d
1859
1860     def test_POST_mkdir_no_replace_field(self): # return value?
1861         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1862                       replace="false")
1863         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1864                   "409 Conflict",
1865                   "There was already a child by that name, and you asked me "
1866                   "to not replace it")
1867         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1868         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1869         return d
1870
1871     def test_POST_mkdir_whendone_field(self):
1872         d = self.POST(self.public_url + "/foo",
1873                       t="mkdir", name="newdir", when_done="/THERE")
1874         d.addBoth(self.shouldRedirect, "/THERE")
1875         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1876         d.addCallback(self.failUnlessNodeKeysAre, [])
1877         return d
1878
1879     def test_POST_mkdir_whendone_queryarg(self):
1880         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1881                       t="mkdir", name="newdir")
1882         d.addBoth(self.shouldRedirect, "/THERE")
1883         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1884         d.addCallback(self.failUnlessNodeKeysAre, [])
1885         return d
1886
1887     def test_POST_bad_t(self):
1888         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1889                              "POST to a directory with bad t=BOGUS",
1890                              self.POST, self.public_url + "/foo", t="BOGUS")
1891         return d
1892
1893     def test_POST_set_children(self):
1894         contents9, n9, newuri9 = self.makefile(9)
1895         contents10, n10, newuri10 = self.makefile(10)
1896         contents11, n11, newuri11 = self.makefile(11)
1897
1898         reqbody = """{
1899                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1900                                                 "size": 0,
1901                                                 "metadata": {
1902                                                   "ctime": 1002777696.7564139,
1903                                                   "mtime": 1002777696.7564139
1904                                                  }
1905                                                } ],
1906                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1907                                                 "size": 1,
1908                                                 "metadata": {
1909                                                   "ctime": 1002777696.7564139,
1910                                                   "mtime": 1002777696.7564139
1911                                                  }
1912                                                } ],
1913                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1914                                                 "size": 2,
1915                                                 "metadata": {
1916                                                   "ctime": 1002777696.7564139,
1917                                                   "mtime": 1002777696.7564139
1918                                                  }
1919                                                } ]
1920                     }""" % (newuri9, newuri10, newuri11)
1921
1922         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1923
1924         d = client.getPage(url, method="POST", postdata=reqbody)
1925         def _then(res):
1926             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1927             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1928             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1929
1930         d.addCallback(_then)
1931         d.addErrback(self.dump_error)
1932         return d
1933
1934     def test_POST_put_uri(self):
1935         contents, n, newuri = self.makefile(8)
1936         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1937         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1938         d.addCallback(lambda res:
1939                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1940                                                       contents))
1941         return d
1942
1943     def test_POST_put_uri_replace(self):
1944         contents, n, newuri = self.makefile(8)
1945         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1946         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1947         d.addCallback(lambda res:
1948                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1949                                                       contents))
1950         return d
1951
1952     def test_POST_put_uri_no_replace_queryarg(self):
1953         contents, n, newuri = self.makefile(8)
1954         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1955                       name="bar.txt", uri=newuri)
1956         d.addBoth(self.shouldFail, error.Error,
1957                   "POST_put_uri_no_replace_queryarg",
1958                   "409 Conflict",
1959                   "There was already a child by that name, and you asked me "
1960                   "to not replace it")
1961         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1962         d.addCallback(self.failUnlessIsBarDotTxt)
1963         return d
1964
1965     def test_POST_put_uri_no_replace_field(self):
1966         contents, n, newuri = self.makefile(8)
1967         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1968                       name="bar.txt", uri=newuri)
1969         d.addBoth(self.shouldFail, error.Error,
1970                   "POST_put_uri_no_replace_field",
1971                   "409 Conflict",
1972                   "There was already a child by that name, and you asked me "
1973                   "to not replace it")
1974         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1975         d.addCallback(self.failUnlessIsBarDotTxt)
1976         return d
1977
1978     def test_POST_delete(self):
1979         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1980         d.addCallback(lambda res: self._foo_node.list())
1981         def _check(children):
1982             self.failIf(u"bar.txt" in children)
1983         d.addCallback(_check)
1984         return d
1985
1986     def test_POST_rename_file(self):
1987         d = self.POST(self.public_url + "/foo", t="rename",
1988                       from_name="bar.txt", to_name='wibble.txt')
1989         d.addCallback(lambda res:
1990                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1991         d.addCallback(lambda res:
1992                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1993         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1994         d.addCallback(self.failUnlessIsBarDotTxt)
1995         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1996         d.addCallback(self.failUnlessIsBarJSON)
1997         return d
1998
1999     def test_POST_rename_file_redundant(self):
2000         d = self.POST(self.public_url + "/foo", t="rename",
2001                       from_name="bar.txt", to_name='bar.txt')
2002         d.addCallback(lambda res:
2003                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2004         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2005         d.addCallback(self.failUnlessIsBarDotTxt)
2006         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2007         d.addCallback(self.failUnlessIsBarJSON)
2008         return d
2009
2010     def test_POST_rename_file_replace(self):
2011         # rename a file and replace a directory with it
2012         d = self.POST(self.public_url + "/foo", t="rename",
2013                       from_name="bar.txt", to_name='empty')
2014         d.addCallback(lambda res:
2015                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2016         d.addCallback(lambda res:
2017                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2018         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2019         d.addCallback(self.failUnlessIsBarDotTxt)
2020         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2021         d.addCallback(self.failUnlessIsBarJSON)
2022         return d
2023
2024     def test_POST_rename_file_no_replace_queryarg(self):
2025         # rename a file and replace a directory with it
2026         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2027                       from_name="bar.txt", to_name='empty')
2028         d.addBoth(self.shouldFail, error.Error,
2029                   "POST_rename_file_no_replace_queryarg",
2030                   "409 Conflict",
2031                   "There was already a child by that name, and you asked me "
2032                   "to not replace it")
2033         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2034         d.addCallback(self.failUnlessIsEmptyJSON)
2035         return d
2036
2037     def test_POST_rename_file_no_replace_field(self):
2038         # rename a file and replace a directory with it
2039         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2040                       from_name="bar.txt", to_name='empty')
2041         d.addBoth(self.shouldFail, error.Error,
2042                   "POST_rename_file_no_replace_field",
2043                   "409 Conflict",
2044                   "There was already a child by that name, and you asked me "
2045                   "to not replace it")
2046         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2047         d.addCallback(self.failUnlessIsEmptyJSON)
2048         return d
2049
2050     def failUnlessIsEmptyJSON(self, res):
2051         data = simplejson.loads(res)
2052         self.failUnlessEqual(data[0], "dirnode", data)
2053         self.failUnlessEqual(len(data[1]["children"]), 0)
2054
2055     def test_POST_rename_file_slash_fail(self):
2056         d = self.POST(self.public_url + "/foo", t="rename",
2057                       from_name="bar.txt", to_name='kirk/spock.txt')
2058         d.addBoth(self.shouldFail, error.Error,
2059                   "test_POST_rename_file_slash_fail",
2060                   "400 Bad Request",
2061                   "to_name= may not contain a slash",
2062                   )
2063         d.addCallback(lambda res:
2064                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2065         return d
2066
2067     def test_POST_rename_dir(self):
2068         d = self.POST(self.public_url, t="rename",
2069                       from_name="foo", to_name='plunk')
2070         d.addCallback(lambda res:
2071                       self.failIfNodeHasChild(self.public_root, u"foo"))
2072         d.addCallback(lambda res:
2073                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2074         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2075         d.addCallback(self.failUnlessIsFooJSON)
2076         return d
2077
2078     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2079         """ If target is not None then the redirection has to go to target.  If
2080         statuscode is not None then the redirection has to be accomplished with
2081         that HTTP status code."""
2082         if not isinstance(res, failure.Failure):
2083             to_where = (target is None) and "somewhere" or ("to " + target)
2084             self.fail("%s: we were expecting to get redirected %s, not get an"
2085                       " actual page: %s" % (which, to_where, res))
2086         res.trap(error.PageRedirect)
2087         if statuscode is not None:
2088             self.failUnlessEqual(res.value.status, statuscode,
2089                                  "%s: not a redirect" % which)
2090         if target is not None:
2091             # the PageRedirect does not seem to capture the uri= query arg
2092             # properly, so we can't check for it.
2093             realtarget = self.webish_url + target
2094             self.failUnlessEqual(res.value.location, realtarget,
2095                                  "%s: wrong target" % which)
2096         return res.value.location
2097
2098     def test_GET_URI_form(self):
2099         base = "/uri?uri=%s" % self._bar_txt_uri
2100         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2101         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2102         d = self.GET(base)
2103         d.addBoth(self.shouldRedirect, targetbase)
2104         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2105         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2106         d.addCallback(lambda res: self.GET(base+"&t=json"))
2107         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2108         d.addCallback(self.log, "about to get file by uri")
2109         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2110         d.addCallback(self.failUnlessIsBarDotTxt)
2111         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2112         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2113                                            followRedirect=True))
2114         d.addCallback(self.failUnlessIsFooJSON)
2115         d.addCallback(self.log, "got dir by uri")
2116
2117         return d
2118
2119     def test_GET_URI_form_bad(self):
2120         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2121                              "400 Bad Request", "GET /uri requires uri=",
2122                              self.GET, "/uri")
2123         return d
2124
2125     def test_GET_rename_form(self):
2126         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2127                      followRedirect=True)
2128         def _check(res):
2129             self.failUnless('name="when_done" value="."' in res, res)
2130             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2131         d.addCallback(_check)
2132         return d
2133
2134     def log(self, res, msg):
2135         #print "MSG: %s  RES: %s" % (msg, res)
2136         log.msg(msg)
2137         return res
2138
2139     def test_GET_URI_URL(self):
2140         base = "/uri/%s" % self._bar_txt_uri
2141         d = self.GET(base)
2142         d.addCallback(self.failUnlessIsBarDotTxt)
2143         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2144         d.addCallback(self.failUnlessIsBarDotTxt)
2145         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2146         d.addCallback(self.failUnlessIsBarDotTxt)
2147         return d
2148
2149     def test_GET_URI_URL_dir(self):
2150         base = "/uri/%s?t=json" % self._foo_uri
2151         d = self.GET(base)
2152         d.addCallback(self.failUnlessIsFooJSON)
2153         return d
2154
2155     def test_GET_URI_URL_missing(self):
2156         base = "/uri/%s" % self._bad_file_uri
2157         d = self.GET(base)
2158         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2159                   http.GONE, response_substring="NotEnoughSharesError")
2160         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2161         # here? we must arrange for a download to fail after target.open()
2162         # has been called, and then inspect the response to see that it is
2163         # shorter than we expected.
2164         return d
2165
2166     def test_PUT_DIRURL_uri(self):
2167         d = self.s.create_empty_dirnode()
2168         def _made_dir(dn):
2169             new_uri = dn.get_uri()
2170             # replace /foo with a new (empty) directory
2171             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2172             d.addCallback(lambda res:
2173                           self.failUnlessEqual(res.strip(), new_uri))
2174             d.addCallback(lambda res:
2175                           self.failUnlessChildURIIs(self.public_root,
2176                                                     u"foo",
2177                                                     new_uri))
2178             return d
2179         d.addCallback(_made_dir)
2180         return d
2181
2182     def test_PUT_DIRURL_uri_noreplace(self):
2183         d = self.s.create_empty_dirnode()
2184         def _made_dir(dn):
2185             new_uri = dn.get_uri()
2186             # replace /foo with a new (empty) directory, but ask that
2187             # replace=false, so it should fail
2188             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2189                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2190                                  self.PUT,
2191                                  self.public_url + "/foo?t=uri&replace=false",
2192                                  new_uri)
2193             d.addCallback(lambda res:
2194                           self.failUnlessChildURIIs(self.public_root,
2195                                                     u"foo",
2196                                                     self._foo_uri))
2197             return d
2198         d.addCallback(_made_dir)
2199         return d
2200
2201     def test_PUT_DIRURL_bad_t(self):
2202         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2203                                  "400 Bad Request", "PUT to a directory",
2204                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2205         d.addCallback(lambda res:
2206                       self.failUnlessChildURIIs(self.public_root,
2207                                                 u"foo",
2208                                                 self._foo_uri))
2209         return d
2210
2211     def test_PUT_NEWFILEURL_uri(self):
2212         contents, n, new_uri = self.makefile(8)
2213         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2214         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2215         d.addCallback(lambda res:
2216                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2217                                                       contents))
2218         return d
2219
2220     def test_PUT_NEWFILEURL_uri_replace(self):
2221         contents, n, new_uri = self.makefile(8)
2222         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2223         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2224         d.addCallback(lambda res:
2225                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2226                                                       contents))
2227         return d
2228
2229     def test_PUT_NEWFILEURL_uri_no_replace(self):
2230         contents, n, new_uri = self.makefile(8)
2231         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2232         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2233                   "409 Conflict",
2234                   "There was already a child by that name, and you asked me "
2235                   "to not replace it")
2236         return d
2237
2238     def test_PUT_NEWFILE_URI(self):
2239         file_contents = "New file contents here\n"
2240         d = self.PUT("/uri", file_contents)
2241         def _check(uri):
2242             self.failUnless(uri in FakeCHKFileNode.all_contents)
2243             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2244                                  file_contents)
2245             return self.GET("/uri/%s" % uri)
2246         d.addCallback(_check)
2247         def _check2(res):
2248             self.failUnlessEqual(res, file_contents)
2249         d.addCallback(_check2)
2250         return d
2251
2252     def test_PUT_NEWFILE_URI_only_PUT(self):
2253         d = self.PUT("/uri?t=bogus", "")
2254         d.addBoth(self.shouldFail, error.Error,
2255                   "PUT_NEWFILE_URI_only_PUT",
2256                   "400 Bad Request",
2257                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2258         return d
2259
2260     def test_PUT_NEWFILE_URI_mutable(self):
2261         file_contents = "New file contents here\n"
2262         d = self.PUT("/uri?mutable=true", file_contents)
2263         def _check_mutable(uri):
2264             uri = uri.strip()
2265             u = IURI(uri)
2266             self.failUnless(IMutableFileURI.providedBy(u))
2267             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2268             n = self.s.create_node_from_uri(uri)
2269             return n.download_best_version()
2270         d.addCallback(_check_mutable)
2271         def _check2_mutable(data):
2272             self.failUnlessEqual(data, file_contents)
2273         d.addCallback(_check2_mutable)
2274         return d
2275
2276         def _check(uri):
2277             self.failUnless(uri in FakeCHKFileNode.all_contents)
2278             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2279                                  file_contents)
2280             return self.GET("/uri/%s" % uri)
2281         d.addCallback(_check)
2282         def _check2(res):
2283             self.failUnlessEqual(res, file_contents)
2284         d.addCallback(_check2)
2285         return d
2286
2287     def test_PUT_mkdir(self):
2288         d = self.PUT("/uri?t=mkdir", "")
2289         def _check(uri):
2290             n = self.s.create_node_from_uri(uri.strip())
2291             d2 = self.failUnlessNodeKeysAre(n, [])
2292             d2.addCallback(lambda res:
2293                            self.GET("/uri/%s?t=json" % uri))
2294             return d2
2295         d.addCallback(_check)
2296         d.addCallback(self.failUnlessIsEmptyJSON)
2297         return d
2298
2299     def test_POST_check(self):
2300         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2301         def _done(res):
2302             # this returns a string form of the results, which are probably
2303             # None since we're using fake filenodes.
2304             # TODO: verify that the check actually happened, by changing
2305             # FakeCHKFileNode to count how many times .check() has been
2306             # called.
2307             pass
2308         d.addCallback(_done)
2309         return d
2310
2311     def test_bad_method(self):
2312         url = self.webish_url + self.public_url + "/foo/bar.txt"
2313         d = self.shouldHTTPError2("test_bad_method",
2314                                   501, "Not Implemented",
2315                                   "I don't know how to treat a BOGUS request.",
2316                                   client.getPage, url, method="BOGUS")
2317         return d
2318
2319     def test_short_url(self):
2320         url = self.webish_url + "/uri"
2321         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2322                                   "I don't know how to treat a DELETE request.",
2323                                   client.getPage, url, method="DELETE")
2324         return d
2325
2326     def test_ophandle_bad(self):
2327         url = self.webish_url + "/operations/bogus?t=status"
2328         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2329                                   "unknown/expired handle 'bogus'",
2330                                   client.getPage, url)
2331         return d
2332
2333     def test_ophandle_cancel(self):
2334         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2335                       followRedirect=True)
2336         d.addCallback(lambda ignored:
2337                       self.GET("/operations/128?t=status&output=JSON"))
2338         def _check1(res):
2339             data = simplejson.loads(res)
2340             self.failUnless("finished" in data, res)
2341             monitor = self.ws.root.child_operations.handles["128"][0]
2342             d = self.POST("/operations/128?t=cancel&output=JSON")
2343             def _check2(res):
2344                 data = simplejson.loads(res)
2345                 self.failUnless("finished" in data, res)
2346                 # t=cancel causes the handle to be forgotten
2347                 self.failUnless(monitor.is_cancelled())
2348             d.addCallback(_check2)
2349             return d
2350         d.addCallback(_check1)
2351         d.addCallback(lambda ignored:
2352                       self.shouldHTTPError2("test_ophandle_cancel",
2353                                             404, "404 Not Found",
2354                                             "unknown/expired handle '128'",
2355                                             self.GET,
2356                                             "/operations/128?t=status&output=JSON"))
2357         return d
2358
2359     def test_ophandle_retainfor(self):
2360         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2361                       followRedirect=True)
2362         d.addCallback(lambda ignored:
2363                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2364         def _check1(res):
2365             data = simplejson.loads(res)
2366             self.failUnless("finished" in data, res)
2367         d.addCallback(_check1)
2368         # the retain-for=0 will cause the handle to be expired very soon
2369         d.addCallback(self.stall, 2.0)
2370         d.addCallback(lambda ignored:
2371                       self.shouldHTTPError2("test_ophandle_retainfor",
2372                                             404, "404 Not Found",
2373                                             "unknown/expired handle '129'",
2374                                             self.GET,
2375                                             "/operations/129?t=status&output=JSON"))
2376         return d
2377
2378     def test_ophandle_release_after_complete(self):
2379         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2380                       followRedirect=True)
2381         d.addCallback(self.wait_for_operation, "130")
2382         d.addCallback(lambda ignored:
2383                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2384         # the release-after-complete=true will cause the handle to be expired
2385         d.addCallback(lambda ignored:
2386                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2387                                             404, "404 Not Found",
2388                                             "unknown/expired handle '130'",
2389                                             self.GET,
2390                                             "/operations/130?t=status&output=JSON"))
2391         return d
2392
2393     def test_incident(self):
2394         d = self.POST("/report_incident", details="eek")
2395         def _done(res):
2396             self.failUnless("Thank you for your report!" in res, res)
2397         d.addCallback(_done)
2398         return d
2399
2400     def test_static(self):
2401         webdir = os.path.join(self.staticdir, "subdir")
2402         fileutil.make_dirs(webdir)
2403         f = open(os.path.join(webdir, "hello.txt"), "wb")
2404         f.write("hello")
2405         f.close()
2406
2407         d = self.GET("/static/subdir/hello.txt")
2408         def _check(res):
2409             self.failUnlessEqual(res, "hello")
2410         d.addCallback(_check)
2411         return d
2412
2413
2414 class Util(unittest.TestCase):
2415     def test_abbreviate_time(self):
2416         self.failUnlessEqual(common.abbreviate_time(None), "")
2417         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2418         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2419         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2420         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2421
2422     def test_abbreviate_rate(self):
2423         self.failUnlessEqual(common.abbreviate_rate(None), "")
2424         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2425         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2426         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2427
2428     def test_abbreviate_size(self):
2429         self.failUnlessEqual(common.abbreviate_size(None), "")
2430         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2431         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2432         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2433         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2434