src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17 import common_util as testutil
  18
  19 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  20 # create a webserver that works against them
  21
  22 class FakeIntroducerClient:
  23     def get_all_connectors(self):
  24         return {}
  25     def get_all_connections_for(self, service_name):
  26         return frozenset()
  27     def get_all_peerids(self):
  28         return frozenset()
  29
  30 class FakeClient(service.MultiService):
  31     nodeid = "fake_nodeid"
  32     nickname = "fake_nickname"
  33     basedir = "fake_basedir"
  34     def get_versions(self):
  35         return {'allmydata': "fake",
  36                 'foolscap': "fake",
  37                 'twisted': "fake",
  38                 'zfec': "fake",
  39                 }
  40     introducer_furl = "None"
  41     introducer_client = FakeIntroducerClient()
  42     _all_upload_status = [upload.UploadStatus()]
  43     _all_download_status = [download.DownloadStatus()]
  44     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  45     _all_publish_statuses = [publish.PublishStatus()]
  46     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  47     convergence = "some random string"
  48
  49     def connected_to_introducer(self):
  50         return False
  51
  52     def get_nickname_for_peerid(self, peerid):
  53         return u"John Doe"
  54
  55     def create_node_from_uri(self, auri):
  56         u = uri.from_string(auri)
  57         if (INewDirectoryURI.providedBy(u)
  58             or IReadonlyNewDirectoryURI.providedBy(u)):
  59             return FakeDirectoryNode(self).init_from_uri(u)
  60         if IFileURI.providedBy(u):
  61             return FakeCHKFileNode(u, self)
  62         assert IMutableFileURI.providedBy(u), u
  63         return FakeMutableFileNode(self).init_from_uri(u)
  64
  65     def create_empty_dirnode(self):
  66         n = FakeDirectoryNode(self)
  67         d = n.create()
  68         d.addCallback(lambda res: n)
  69         return d
  70
  71     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  72     def create_mutable_file(self, contents=""):
  73         n = FakeMutableFileNode(self)
  74         return n.create(contents)
  75
  76     def upload(self, uploadable):
  77         d = uploadable.get_size()
  78         d.addCallback(lambda size: uploadable.read(size))
  79         def _got_data(datav):
  80             data = "".join(datav)
  81             n = create_chk_filenode(self, data)
  82             results = upload.UploadResults()
  83             results.uri = n.get_uri()
  84             return results
  85         d.addCallback(_got_data)
  86         return d
  87
  88     def list_all_upload_statuses(self):
  89         return self._all_upload_status
  90     def list_all_download_statuses(self):
  91         return self._all_download_status
  92     def list_all_mapupdate_statuses(self):
  93         return self._all_mapupdate_statuses
  94     def list_all_publish_statuses(self):
  95         return self._all_publish_statuses
  96     def list_all_retrieve_statuses(self):
  97         return self._all_retrieve_statuses
  98     def list_all_helper_statuses(self):
  99         return []
 100
 101 class MyGetter(client.HTTPPageGetter):
 102     handleStatus_206 = lambda self: self.handleStatus_200()
 103
 104 class HTTPClientHEADFactory(client.HTTPClientFactory):
 105     protocol = MyGetter
 106
 107     def noPage(self, reason):
 108         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 109         # exception when the response to a HEAD request had no body (when in
 110         # fact they are defined to never have a body). This was fixed in
 111         # Twisted-8.0 . To work around this, we catch the
 112         # PartialDownloadError and make it disappear.
 113         if (reason.check(client.PartialDownloadError)
 114             and self.method.upper() == "HEAD"):
 115             self.page("")
 116             return
 117         return client.HTTPClientFactory.noPage(self, reason)
 118
 119 class HTTPClientGETFactory(client.HTTPClientFactory):
 120     protocol = MyGetter
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.ws = s = webish.WebishServer("0")
 127         s.setServiceParent(self.s)
 128         self.webish_port = port = s.listener._port.getHost().port
 129         self.webish_url = "http://localhost:%d" % port
 130
 131         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 132         d = defer.DeferredList(l)
 133         def _then(res):
 134             self.public_root = res[0][1]
 135             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 136             self.public_url = "/uri/" + self.public_root.get_uri()
 137             self.private_root = res[1][1]
 138
 139             foo = res[2][1]
 140             self._foo_node = foo
 141             self._foo_uri = foo.get_uri()
 142             self._foo_readonly_uri = foo.get_readonly_uri()
 143             # NOTE: we ignore the deferred on all set_uri() calls, because we
 144             # know the fake nodes do these synchronously
 145             self.public_root.set_uri(u"foo", foo.get_uri())
 146
 147             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 148             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 149
 150             foo.set_uri(u"empty", res[3][1].get_uri())
 151             sub_uri = res[4][1].get_uri()
 152             self._sub_uri = sub_uri
 153             foo.set_uri(u"sub", sub_uri)
 154             sub = self.s.create_node_from_uri(sub_uri)
 155
 156             _ign, n, blocking_uri = self.makefile(1)
 157             foo.set_uri(u"blockingfile", blocking_uri)
 158
 159             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 160             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 161             # still think of it as an umlaut
 162             foo.set_uri(unicode_filename, self._bar_txt_uri)
 163
 164             _ign, n, baz_file = self.makefile(2)
 165             sub.set_uri(u"baz.txt", baz_file)
 166
 167             _ign, n, self._bad_file_uri = self.makefile(3)
 168             # this uri should not be downloadable
 169             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 170
 171             rodir = res[5][1]
 172             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 173             rodir.set_uri(u"nor", baz_file)
 174
 175             # public/
 176             # public/foo/
 177             # public/foo/bar.txt
 178             # public/foo/blockingfile
 179             # public/foo/empty/
 180             # public/foo/sub/
 181             # public/foo/sub/baz.txt
 182             # public/reedownlee/
 183             # public/reedownlee/nor
 184             self.NEWFILE_CONTENTS = "newfile contents\n"
 185
 186             return foo.get_metadata_for(u"bar.txt")
 187         d.addCallback(_then)
 188         def _got_metadata(metadata):
 189             self._bar_txt_metadata = metadata
 190         d.addCallback(_got_metadata)
 191         return d
 192
 193     def makefile(self, number):
 194         contents = "contents of file %s\n" % number
 195         n = create_chk_filenode(self.s, contents)
 196         return contents, n, n.get_uri()
 197
 198     def tearDown(self):
 199         return self.s.stopService()
 200
 201     def failUnlessIsBarDotTxt(self, res):
 202         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 203
 204     def failUnlessIsBarJSON(self, res):
 205         data = simplejson.loads(res)
 206         self.failUnless(isinstance(data, list))
 207         self.failUnlessEqual(data[0], u"filenode")
 208         self.failUnless(isinstance(data[1], dict))
 209         self.failIf(data[1]["mutable"])
 210         self.failIf("rw_uri" in data[1]) # immutable
 211         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 212         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 213
 214     def failUnlessIsFooJSON(self, res):
 215         data = simplejson.loads(res)
 216         self.failUnless(isinstance(data, list))
 217         self.failUnlessEqual(data[0], "dirnode", res)
 218         self.failUnless(isinstance(data[1], dict))
 219         self.failUnless(data[1]["mutable"])
 220         self.failUnless("rw_uri" in data[1]) # mutable
 221         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 222         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 223
 224         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 225         self.failUnlessEqual(kidnames,
 226                              [u"bar.txt", u"blockingfile", u"empty",
 227                               u"n\u00fc.txt", u"sub"])
 228         kids = dict( [(unicode(name),value)
 229                       for (name,value)
 230                       in data[1]["children"].iteritems()] )
 231         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 232         self.failUnless("metadata" in kids[u"sub"][1])
 233         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 234         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 235         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 236         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 237         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 238         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 239                              self._bar_txt_metadata["ctime"])
 240         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 241                              self._bar_txt_uri)
 242
 243     def GET(self, urlpath, followRedirect=False, return_response=False,
 244             **kwargs):
 245         # if return_response=True, this fires with (data, statuscode,
 246         # respheaders) instead of just data.
 247         assert not isinstance(urlpath, unicode)
 248         url = self.webish_url + urlpath
 249         factory = HTTPClientGETFactory(url, method="GET",
 250                                        followRedirect=followRedirect, **kwargs)
 251         reactor.connectTCP("localhost", self.webish_port, factory)
 252         d = factory.deferred
 253         def _got_data(data):
 254             return (data, factory.status, factory.response_headers)
 255         if return_response:
 256             d.addCallback(_got_data)
 257         return factory.deferred
 258
 259     def HEAD(self, urlpath, return_response=False, **kwargs):
 260         # this requires some surgery, because twisted.web.client doesn't want
 261         # to give us back the response headers.
 262         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 263         reactor.connectTCP("localhost", self.webish_port, factory)
 264         d = factory.deferred
 265         def _got_data(data):
 266             return (data, factory.status, factory.response_headers)
 267         if return_response:
 268             d.addCallback(_got_data)
 269         return factory.deferred
 270
 271     def PUT(self, urlpath, data, **kwargs):
 272         url = self.webish_url + urlpath
 273         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 274
 275     def DELETE(self, urlpath):
 276         url = self.webish_url + urlpath
 277         return client.getPage(url, method="DELETE")
 278
 279     def POST(self, urlpath, followRedirect=False, **fields):
 280         url = self.webish_url + urlpath
 281         sepbase = "boogabooga"
 282         sep = "--" + sepbase
 283         form = []
 284         form.append(sep)
 285         form.append('Content-Disposition: form-data; name="_charset"')
 286         form.append('')
 287         form.append('UTF-8')
 288         form.append(sep)
 289         for name, value in fields.iteritems():
 290             if isinstance(value, tuple):
 291                 filename, value = value
 292                 form.append('Content-Disposition: form-data; name="%s"; '
 293                             'filename="%s"' % (name, filename.encode("utf-8")))
 294             else:
 295                 form.append('Content-Disposition: form-data; name="%s"' % name)
 296             form.append('')
 297             if isinstance(value, unicode):
 298                 value = value.encode("utf-8")
 299             else:
 300                 value = str(value)
 301             assert isinstance(value, str)
 302             form.append(value)
 303             form.append(sep)
 304         form[-1] += "--"
 305         body = "\r\n".join(form) + "\r\n"
 306         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 307                    }
 308         return client.getPage(url, method="POST", postdata=body,
 309                               headers=headers, followRedirect=followRedirect)
 310
 311     def shouldFail(self, res, expected_failure, which,
 312                    substring=None, response_substring=None):
 313         if isinstance(res, failure.Failure):
 314             res.trap(expected_failure)
 315             if substring:
 316                 self.failUnless(substring in str(res),
 317                                 "substring '%s' not in '%s'"
 318                                 % (substring, str(res)))
 319             if response_substring:
 320                 self.failUnless(response_substring in res.value.response,
 321                                 "response substring '%s' not in '%s'"
 322                                 % (response_substring, res.value.response))
 323         else:
 324             self.fail("%s was supposed to raise %s, not get '%s'" %
 325                       (which, expected_failure, res))
 326
 327     def shouldFail2(self, expected_failure, which, substring,
 328                     response_substring,
 329                     callable, *args, **kwargs):
 330         assert substring is None or isinstance(substring, str)
 331         assert response_substring is None or isinstance(response_substring, str)
 332         d = defer.maybeDeferred(callable, *args, **kwargs)
 333         def done(res):
 334             if isinstance(res, failure.Failure):
 335                 res.trap(expected_failure)
 336                 if substring:
 337                     self.failUnless(substring in str(res),
 338                                     "%s: substring '%s' not in '%s'"
 339                                     % (which, substring, str(res)))
 340                 if response_substring:
 341                     self.failUnless(response_substring in res.value.response,
 342                                     "%s: response substring '%s' not in '%s'"
 343                                     % (which,
 344                                        response_substring, res.value.response))
 345             else:
 346                 self.fail("%s was supposed to raise %s, not get '%s'" %
 347                           (which, expected_failure, res))
 348         d.addBoth(done)
 349         return d
 350
 351     def should404(self, res, which):
 352         if isinstance(res, failure.Failure):
 353             res.trap(error.Error)
 354             self.failUnlessEqual(res.value.status, "404")
 355         else:
 356             self.fail("%s was supposed to Error(404), not get '%s'" %
 357                       (which, res))
 358
 359     def shouldHTTPError(self, res, which, code=None, substring=None,
 360                         response_substring=None):
 361         if isinstance(res, failure.Failure):
 362             res.trap(error.Error)
 363             if code is not None:
 364                 self.failUnlessEqual(res.value.status, str(code))
 365             if substring:
 366                 self.failUnless(substring in str(res),
 367                                 "substring '%s' not in '%s'"
 368                                 % (substring, str(res)))
 369             if response_substring:
 370                 self.failUnless(response_substring in res.value.response,
 371                                 "response substring '%s' not in '%s'"
 372                                 % (response_substring, res.value.response))
 373         else:
 374             self.fail("%s was supposed to Error(%s), not get '%s'" %
 375                       (which, code, res))
 376
 377     def shouldHTTPError2(self, which,
 378                          code=None, substring=None, response_substring=None,
 379                          callable=None, *args, **kwargs):
 380         assert substring is None or isinstance(substring, str)
 381         assert callable
 382         d = defer.maybeDeferred(callable, *args, **kwargs)
 383         d.addBoth(self.shouldHTTPError, which,
 384                   code, substring, response_substring)
 385         return d
 386
 387
 388 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 389     def test_create(self):
 390         pass
 391
 392     def test_welcome(self):
 393         d = self.GET("/")
 394         def _check(res):
 395             self.failUnless('Welcome To AllMyData' in res)
 396             self.failUnless('Tahoe' in res)
 397
 398             self.s.basedir = 'web/test_welcome'
 399             fileutil.make_dirs("web/test_welcome")
 400             fileutil.make_dirs("web/test_welcome/private")
 401             return self.GET("/")
 402         d.addCallback(_check)
 403         return d
 404
 405     def test_provisioning_math(self):
 406         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 407         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 408         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 409         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 410         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 411
 412     def test_provisioning(self):
 413         d = self.GET("/provisioning/")
 414         def _check(res):
 415             self.failUnless('Tahoe Provisioning Tool' in res)
 416             fields = {'filled': True,
 417                       "num_users": int(50e3),
 418                       "files_per_user": 1000,
 419                       "space_per_user": int(1e9),
 420                       "sharing_ratio": 1.0,
 421                       "encoding_parameters": "3-of-10-5",
 422                       "num_servers": 30,
 423                       "ownership_mode": "A",
 424                       "download_rate": 100,
 425                       "upload_rate": 10,
 426                       "delete_rate": 10,
 427                       "lease_timer": 7,
 428                       }
 429             return self.POST("/provisioning/", **fields)
 430
 431         d.addCallback(_check)
 432         def _check2(res):
 433             self.failUnless('Tahoe Provisioning Tool' in res)
 434             self.failUnless("Share space consumed: 167.01TB" in res)
 435
 436             fields = {'filled': True,
 437                       "num_users": int(50e6),
 438                       "files_per_user": 1000,
 439                       "space_per_user": int(5e9),
 440                       "sharing_ratio": 1.0,
 441                       "encoding_parameters": "25-of-100-50",
 442                       "num_servers": 30000,
 443                       "ownership_mode": "E",
 444                       "drive_failure_model": "U",
 445                       "drive_size": 1000,
 446                       "download_rate": 1000,
 447                       "upload_rate": 100,
 448                       "delete_rate": 100,
 449                       "lease_timer": 7,
 450                       }
 451             return self.POST("/provisioning/", **fields)
 452         d.addCallback(_check2)
 453         def _check3(res):
 454             self.failUnless("Share space consumed: huge!" in res)
 455             fields = {'filled': True}
 456             return self.POST("/provisioning/", **fields)
 457         d.addCallback(_check3)
 458         def _check4(res):
 459             self.failUnless("Share space consumed:" in res)
 460         d.addCallback(_check4)
 461         return d
 462
 463     def test_status(self):
 464         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 465         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 466         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 467         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 468         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 469         d = self.GET("/status", followRedirect=True)
 470         def _check(res):
 471             self.failUnless('Upload and Download Status' in res, res)
 472             self.failUnless('"down-%d"' % dl_num in res, res)
 473             self.failUnless('"up-%d"' % ul_num in res, res)
 474             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 475             self.failUnless('"publish-%d"' % pub_num in res, res)
 476             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 477         d.addCallback(_check)
 478         d.addCallback(lambda res: self.GET("/status/?t=json"))
 479         def _check_json(res):
 480             data = simplejson.loads(res)
 481             self.failUnless(isinstance(data, dict))
 482             active = data["active"]
 483             # TODO: test more. We need a way to fake an active operation
 484             # here.
 485         d.addCallback(_check_json)
 486
 487         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 488         def _check_dl(res):
 489             self.failUnless("File Download Status" in res, res)
 490         d.addCallback(_check_dl)
 491         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 492         def _check_ul(res):
 493             self.failUnless("File Upload Status" in res, res)
 494         d.addCallback(_check_ul)
 495         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 496         def _check_mapupdate(res):
 497             self.failUnless("Mutable File Servermap Update Status" in res, res)
 498         d.addCallback(_check_mapupdate)
 499         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 500         def _check_publish(res):
 501             self.failUnless("Mutable File Publish Status" in res, res)
 502         d.addCallback(_check_publish)
 503         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 504         def _check_retrieve(res):
 505             self.failUnless("Mutable File Retrieve Status" in res, res)
 506         d.addCallback(_check_retrieve)
 507
 508         return d
 509
 510     def test_status_numbers(self):
 511         drrm = status.DownloadResultsRendererMixin()
 512         self.failUnlessEqual(drrm.render_time(None, None), "")
 513         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 514         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 515         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 517         self.failUnlessEqual(drrm.render_rate(None, None), "")
 518         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 519         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 521
 522         urrm = status.UploadResultsRendererMixin()
 523         self.failUnlessEqual(urrm.render_time(None, None), "")
 524         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 525         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 526         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 528         self.failUnlessEqual(urrm.render_rate(None, None), "")
 529         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 530         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 532
 533     def test_GET_FILEURL(self):
 534         d = self.GET(self.public_url + "/foo/bar.txt")
 535         d.addCallback(self.failUnlessIsBarDotTxt)
 536         return d
 537
 538     def test_GET_FILEURL_range(self):
 539         headers = {"range": "bytes=1-10"}
 540         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 541                      return_response=True)
 542         def _got((res, status, headers)):
 543             self.failUnlessEqual(int(status), 206)
 544             self.failUnless(headers.has_key("content-range"))
 545             self.failUnlessEqual(headers["content-range"][0],
 546                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 547             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 548         d.addCallback(_got)
 549         return d
 550
 551     def test_HEAD_FILEURL_range(self):
 552         headers = {"range": "bytes=1-10"}
 553         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 554                      return_response=True)
 555         def _got((res, status, headers)):
 556             self.failUnlessEqual(res, "")
 557             self.failUnlessEqual(int(status), 206)
 558             self.failUnless(headers.has_key("content-range"))
 559             self.failUnlessEqual(headers["content-range"][0],
 560                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 561         d.addCallback(_got)
 562         return d
 563
 564     def test_GET_FILEURL_range_bad(self):
 565         headers = {"range": "BOGUS=fizbop-quarnak"}
 566         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 567                              "400 Bad Request",
 568                              "Syntactically invalid http range header",
 569                              self.GET, self.public_url + "/foo/bar.txt",
 570                              headers=headers)
 571         return d
 572
 573     def test_HEAD_FILEURL(self):
 574         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 575         def _got((res, status, headers)):
 576             self.failUnlessEqual(res, "")
 577             self.failUnlessEqual(headers["content-length"][0],
 578                                  str(len(self.BAR_CONTENTS)))
 579             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 580         d.addCallback(_got)
 581         return d
 582
 583     def test_GET_FILEURL_named(self):
 584         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 585         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 586         d = self.GET(base + "/@@name=/blah.txt")
 587         d.addCallback(self.failUnlessIsBarDotTxt)
 588         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 589         d.addCallback(self.failUnlessIsBarDotTxt)
 590         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 591         d.addCallback(self.failUnlessIsBarDotTxt)
 592         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 593         d.addCallback(self.failUnlessIsBarDotTxt)
 594         save_url = base + "?save=true&filename=blah.txt"
 595         d.addCallback(lambda res: self.GET(save_url))
 596         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 597         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 598         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 599         u_url = base + "?save=true&filename=" + u_fn_e
 600         d.addCallback(lambda res: self.GET(u_url))
 601         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 602         return d
 603
 604     def test_PUT_FILEURL_named_bad(self):
 605         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 606         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 607                              "400 Bad Request",
 608                              "/file can only be used with GET or HEAD",
 609                              self.PUT, base + "/@@name=/blah.txt", "")
 610         return d
 611
 612     def test_GET_DIRURL_named_bad(self):
 613         base = "/file/%s" % urllib.quote(self._foo_uri)
 614         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 615                              "400 Bad Request",
 616                              "is not a file-cap",
 617                              self.GET, base + "/@@name=/blah.txt")
 618         return d
 619
 620     def test_GET_slash_file_bad(self):
 621         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 622                              "404 Not Found",
 623                              "/file must be followed by a file-cap and a name",
 624                              self.GET, "/file")
 625         return d
 626
 627     def test_GET_unhandled_URI_named(self):
 628         contents, n, newuri = self.makefile(12)
 629         verifier_cap = n.get_verifier().to_string()
 630         base = "/file/%s" % urllib.quote(verifier_cap)
 631         # client.create_node_from_uri() can't handle verify-caps
 632         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 633                              "400 Bad Request",
 634                              "is not a valid file- or directory- cap",
 635                              self.GET, base)
 636         return d
 637
 638     def test_GET_unhandled_URI(self):
 639         contents, n, newuri = self.makefile(12)
 640         verifier_cap = n.get_verifier().to_string()
 641         base = "/uri/%s" % urllib.quote(verifier_cap)
 642         # client.create_node_from_uri() can't handle verify-caps
 643         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 644                              "400 Bad Request",
 645                              "is not a valid file- or directory- cap",
 646                              self.GET, base)
 647         return d
 648
 649     def test_GET_FILE_URI(self):
 650         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 651         d = self.GET(base)
 652         d.addCallback(self.failUnlessIsBarDotTxt)
 653         return d
 654
 655     def test_GET_FILE_URI_badchild(self):
 656         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 657         errmsg = "Files have no children, certainly not named 'boguschild'"
 658         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 659                              "400 Bad Request", errmsg,
 660                              self.GET, base)
 661         return d
 662
 663     def test_PUT_FILE_URI_badchild(self):
 664         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 665         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 666         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 667                              "400 Bad Request", errmsg,
 668                              self.PUT, base, "")
 669         return d
 670
 671     def test_GET_FILEURL_save(self):
 672         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 673         # TODO: look at the headers, expect a Content-Disposition: attachment
 674         # header.
 675         d.addCallback(self.failUnlessIsBarDotTxt)
 676         return d
 677
 678     def test_GET_FILEURL_missing(self):
 679         d = self.GET(self.public_url + "/foo/missing")
 680         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 681         return d
 682
 683     def test_PUT_NEWFILEURL(self):
 684         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 685         # TODO: we lose the response code, so we can't check this
 686         #self.failUnlessEqual(responsecode, 201)
 687         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 688         d.addCallback(lambda res:
 689                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 690                                                       self.NEWFILE_CONTENTS))
 691         return d
 692
 693     def test_PUT_NEWFILEURL_range_bad(self):
 694         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 695         target = self.public_url + "/foo/new.txt"
 696         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 697                              "501 Not Implemented",
 698                              "Content-Range in PUT not yet supported",
 699                              # (and certainly not for immutable files)
 700                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 701                              headers=headers)
 702         d.addCallback(lambda res:
 703                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 704         return d
 705
 706     def test_PUT_NEWFILEURL_mutable(self):
 707         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 708                      self.NEWFILE_CONTENTS)
 709         # TODO: we lose the response code, so we can't check this
 710         #self.failUnlessEqual(responsecode, 201)
 711         def _check_uri(res):
 712             u = uri.from_string_mutable_filenode(res)
 713             self.failUnless(u.is_mutable())
 714             self.failIf(u.is_readonly())
 715             return res
 716         d.addCallback(_check_uri)
 717         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 718         d.addCallback(lambda res:
 719                       self.failUnlessMutableChildContentsAre(self._foo_node,
 720                                                              u"new.txt",
 721                                                              self.NEWFILE_CONTENTS))
 722         return d
 723
 724     def test_PUT_NEWFILEURL_mutable_toobig(self):
 725         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 726                              "413 Request Entity Too Large",
 727                              "SDMF is limited to one segment, and 10001 > 10000",
 728                              self.PUT,
 729                              self.public_url + "/foo/new.txt?mutable=true",
 730                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 731         return d
 732
 733     def test_PUT_NEWFILEURL_replace(self):
 734         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 735         # TODO: we lose the response code, so we can't check this
 736         #self.failUnlessEqual(responsecode, 200)
 737         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 738         d.addCallback(lambda res:
 739                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 740                                                       self.NEWFILE_CONTENTS))
 741         return d
 742
 743     def test_PUT_NEWFILEURL_bad_t(self):
 744         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 745                              "PUT to a file: bad t=bogus",
 746                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 747                              "contents")
 748         return d
 749
 750     def test_PUT_NEWFILEURL_no_replace(self):
 751         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 752                      self.NEWFILE_CONTENTS)
 753         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 754                   "409 Conflict",
 755                   "There was already a child by that name, and you asked me "
 756                   "to not replace it")
 757         return d
 758
 759     def test_PUT_NEWFILEURL_mkdirs(self):
 760         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 761         fn = self._foo_node
 762         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 763         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 764         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 765         d.addCallback(lambda res:
 766                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 767                                                       self.NEWFILE_CONTENTS))
 768         return d
 769
 770     def test_PUT_NEWFILEURL_blocked(self):
 771         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 772                      self.NEWFILE_CONTENTS)
 773         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 774                   "409 Conflict",
 775                   "Unable to create directory 'blockingfile': a file was in the way")
 776         return d
 777
 778     def test_DELETE_FILEURL(self):
 779         d = self.DELETE(self.public_url + "/foo/bar.txt")
 780         d.addCallback(lambda res:
 781                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 782         return d
 783
 784     def test_DELETE_FILEURL_missing(self):
 785         d = self.DELETE(self.public_url + "/foo/missing")
 786         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 787         return d
 788
 789     def test_DELETE_FILEURL_missing2(self):
 790         d = self.DELETE(self.public_url + "/missing/missing")
 791         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 792         return d
 793
 794     def test_GET_FILEURL_json(self):
 795         # twisted.web.http.parse_qs ignores any query args without an '=', so
 796         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 797         # instead. This may make it tricky to emulate the S3 interface
 798         # completely.
 799         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 800         d.addCallback(self.failUnlessIsBarJSON)
 801         return d
 802
 803     def test_GET_FILEURL_json_missing(self):
 804         d = self.GET(self.public_url + "/foo/missing?json")
 805         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 806         return d
 807
 808     def test_GET_FILEURL_uri(self):
 809         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 810         def _check(res):
 811             self.failUnlessEqual(res, self._bar_txt_uri)
 812         d.addCallback(_check)
 813         d.addCallback(lambda res:
 814                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 815         def _check2(res):
 816             # for now, for files, uris and readonly-uris are the same
 817             self.failUnlessEqual(res, self._bar_txt_uri)
 818         d.addCallback(_check2)
 819         return d
 820
 821     def test_GET_FILEURL_badtype(self):
 822         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 823                                   "bad t=bogus",
 824                                   self.GET,
 825                                   self.public_url + "/foo/bar.txt?t=bogus")
 826         return d
 827
 828     def test_GET_FILEURL_uri_missing(self):
 829         d = self.GET(self.public_url + "/foo/missing?t=uri")
 830         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 831         return d
 832
 833     def test_GET_DIRURL(self):
 834         # the addSlash means we get a redirect here
 835         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 836         ROOT = "../../.."
 837         d = self.GET(self.public_url + "/foo", followRedirect=True)
 838         def _check(res):
 839             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 840                             in res, res)
 841             # the FILE reference points to a URI, but it should end in bar.txt
 842             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 843                        (ROOT, urllib.quote(self._bar_txt_uri)))
 844             get_bar = "".join([r'<td>',
 845                                r'<a href="%s">bar.txt</a>' % bar_url,
 846                                r'</td>',
 847                                r'\s+<td>FILE</td>',
 848                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 849                                ])
 850             self.failUnless(re.search(get_bar, res), res)
 851             for line in res.split("\n"):
 852                 # find the line that contains the delete button for bar.txt
 853                 if ("form action" in line and
 854                     'value="delete"' in line and
 855                     'value="bar.txt"' in line):
 856                     # the form target should use a relative URL
 857                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 858                     self.failUnless(('action="%s"' % foo_url) in line, line)
 859                     # and the when_done= should too
 860                     #done_url = urllib.quote(???)
 861                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 862                     #                in line, line)
 863                     break
 864             else:
 865                 self.fail("unable to find delete-bar.txt line", res)
 866
 867             # the DIR reference just points to a URI
 868             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 869             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 870                        + r'\s+<td>DIR</td>')
 871             self.failUnless(re.search(get_sub, res), res)
 872         d.addCallback(_check)
 873
 874         # look at a directory which is readonly
 875         d.addCallback(lambda res:
 876                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 877         def _check2(res):
 878             self.failUnless("(readonly)" in res, res)
 879             self.failIf("Upload a file" in res, res)
 880         d.addCallback(_check2)
 881
 882         # and at a directory that contains a readonly directory
 883         d.addCallback(lambda res:
 884                       self.GET(self.public_url, followRedirect=True))
 885         def _check3(res):
 886             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 887                                       '</td>\s+<td>DIR-RO</td>', res))
 888         d.addCallback(_check3)
 889
 890         return d
 891
 892     def test_GET_DIRURL_badtype(self):
 893         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 894                                   400, "Bad Request",
 895                                   "bad t=bogus",
 896                                   self.GET,
 897                                   self.public_url + "/foo?t=bogus")
 898         return d
 899
 900     def test_GET_DIRURL_json(self):
 901         d = self.GET(self.public_url + "/foo?t=json")
 902         d.addCallback(self.failUnlessIsFooJSON)
 903         return d
 904
 905
 906     def test_POST_DIRURL_manifest_no_ophandle(self):
 907         d = self.shouldFail2(error.Error,
 908                              "test_POST_DIRURL_manifest_no_ophandle",
 909                              "400 Bad Request",
 910                              "slow operation requires ophandle=",
 911                              self.POST, self.public_url, t="start-manifest")
 912         return d
 913
 914     def test_POST_DIRURL_manifest(self):
 915         d = defer.succeed(None)
 916         def getman(ignored, output):
 917             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 918                           followRedirect=True)
 919             d.addCallback(self.wait_for_operation, "125")
 920             d.addCallback(self.get_operation_results, "125", output)
 921             return d
 922         d.addCallback(getman, None)
 923         def _got_html(manifest):
 924             self.failUnless("Manifest of SI=" in manifest)
 925             self.failUnless("<td>sub</td>" in manifest)
 926             self.failUnless(self._sub_uri in manifest)
 927             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 928         d.addCallback(_got_html)
 929
 930         # both t=status and unadorned GET should be identical
 931         d.addCallback(lambda res: self.GET("/operations/125"))
 932         d.addCallback(_got_html)
 933
 934         d.addCallback(getman, "html")
 935         d.addCallback(_got_html)
 936         d.addCallback(getman, "text")
 937         def _got_text(manifest):
 938             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 939             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 940         d.addCallback(_got_text)
 941         d.addCallback(getman, "JSON")
 942         def _got_json(manifest):
 943             data = manifest["manifest"]
 944             got = {}
 945             for (path_list, cap) in data:
 946                 got[tuple(path_list)] = cap
 947             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 948             self.failUnless((u"sub",u"baz.txt") in got)
 949         d.addCallback(_got_json)
 950         return d
 951
 952     def test_POST_DIRURL_deepsize_no_ophandle(self):
 953         d = self.shouldFail2(error.Error,
 954                              "test_POST_DIRURL_deepsize_no_ophandle",
 955                              "400 Bad Request",
 956                              "slow operation requires ophandle=",
 957                              self.POST, self.public_url, t="start-deep-size")
 958         return d
 959
 960     def test_POST_DIRURL_deepsize(self):
 961         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 962                       followRedirect=True)
 963         d.addCallback(self.wait_for_operation, "126")
 964         d.addCallback(self.get_operation_results, "126", "json")
 965         def _got_json(data):
 966             self.failUnlessEqual(data["finished"], True)
 967             size = data["size"]
 968             self.failUnless(size > 1000)
 969         d.addCallback(_got_json)
 970         d.addCallback(self.get_operation_results, "126", "text")
 971         def _got_text(res):
 972             mo = re.search(r'^size: (\d+)$', res, re.M)
 973             self.failUnless(mo, res)
 974             size = int(mo.group(1))
 975             # with directories, the size varies.
 976             self.failUnless(size > 1000)
 977         d.addCallback(_got_text)
 978         return d
 979
 980     def test_POST_DIRURL_deepstats_no_ophandle(self):
 981         d = self.shouldFail2(error.Error,
 982                              "test_POST_DIRURL_deepstats_no_ophandle",
 983                              "400 Bad Request",
 984                              "slow operation requires ophandle=",
 985                              self.POST, self.public_url, t="start-deep-stats")
 986         return d
 987
 988     def test_POST_DIRURL_deepstats(self):
 989         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 990                       followRedirect=True)
 991         d.addCallback(self.wait_for_operation, "127")
 992         d.addCallback(self.get_operation_results, "127", "json")
 993         def _got_json(stats):
 994             expected = {"count-immutable-files": 3,
 995                         "count-mutable-files": 0,
 996                         "count-literal-files": 0,
 997                         "count-files": 3,
 998                         "count-directories": 3,
 999                         "size-immutable-files": 57,
1000                         "size-literal-files": 0,
1001                         #"size-directories": 1912, # varies
1002                         #"largest-directory": 1590,
1003                         "largest-directory-children": 5,
1004                         "largest-immutable-file": 19,
1005                         }
1006             for k,v in expected.iteritems():
1007                 self.failUnlessEqual(stats[k], v,
1008                                      "stats[%s] was %s, not %s" %
1009                                      (k, stats[k], v))
1010             self.failUnlessEqual(stats["size-files-histogram"],
1011                                  [ [11, 31, 3] ])
1012         d.addCallback(_got_json)
1013         return d
1014
1015     def test_GET_DIRURL_uri(self):
1016         d = self.GET(self.public_url + "/foo?t=uri")
1017         def _check(res):
1018             self.failUnlessEqual(res, self._foo_uri)
1019         d.addCallback(_check)
1020         return d
1021
1022     def test_GET_DIRURL_readonly_uri(self):
1023         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1024         def _check(res):
1025             self.failUnlessEqual(res, self._foo_readonly_uri)
1026         d.addCallback(_check)
1027         return d
1028
1029     def test_PUT_NEWDIRURL(self):
1030         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1031         d.addCallback(lambda res:
1032                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1033         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1034         d.addCallback(self.failUnlessNodeKeysAre, [])
1035         return d
1036
1037     def test_PUT_NEWDIRURL_exists(self):
1038         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1039         d.addCallback(lambda res:
1040                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1041         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1042         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1043         return d
1044
1045     def test_PUT_NEWDIRURL_blocked(self):
1046         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1047                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1048                              self.PUT,
1049                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1050         d.addCallback(lambda res:
1051                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1052         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1053         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1054         return d
1055
1056     def test_PUT_NEWDIRURL_mkdir_p(self):
1057         d = defer.succeed(None)
1058         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1059         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1060         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1061         def mkdir_p(mkpnode):
1062             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1063             d = self.POST(url)
1064             def made_subsub(ssuri):
1065                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1066                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1067                 d = self.POST(url)
1068                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1069                 return d
1070             d.addCallback(made_subsub)
1071             return d
1072         d.addCallback(mkdir_p)
1073         return d
1074
1075     def test_PUT_NEWDIRURL_mkdirs(self):
1076         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1077         d.addCallback(lambda res:
1078                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1079         d.addCallback(lambda res:
1080                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1081         d.addCallback(lambda res:
1082                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1083         d.addCallback(self.failUnlessNodeKeysAre, [])
1084         return d
1085
1086     def test_DELETE_DIRURL(self):
1087         d = self.DELETE(self.public_url + "/foo")
1088         d.addCallback(lambda res:
1089                       self.failIfNodeHasChild(self.public_root, u"foo"))
1090         return d
1091
1092     def test_DELETE_DIRURL_missing(self):
1093         d = self.DELETE(self.public_url + "/foo/missing")
1094         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1095         d.addCallback(lambda res:
1096                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1097         return d
1098
1099     def test_DELETE_DIRURL_missing2(self):
1100         d = self.DELETE(self.public_url + "/missing")
1101         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1102         return d
1103
1104     def dump_root(self):
1105         print "NODEWALK"
1106         w = webish.DirnodeWalkerMixin()
1107         def visitor(childpath, childnode, metadata):
1108             print childpath
1109         d = w.walk(self.public_root, visitor)
1110         return d
1111
1112     def failUnlessNodeKeysAre(self, node, expected_keys):
1113         for k in expected_keys:
1114             assert isinstance(k, unicode)
1115         d = node.list()
1116         def _check(children):
1117             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1118         d.addCallback(_check)
1119         return d
1120     def failUnlessNodeHasChild(self, node, name):
1121         assert isinstance(name, unicode)
1122         d = node.list()
1123         def _check(children):
1124             self.failUnless(name in children)
1125         d.addCallback(_check)
1126         return d
1127     def failIfNodeHasChild(self, node, name):
1128         assert isinstance(name, unicode)
1129         d = node.list()
1130         def _check(children):
1131             self.failIf(name in children)
1132         d.addCallback(_check)
1133         return d
1134
1135     def failUnlessChildContentsAre(self, node, name, expected_contents):
1136         assert isinstance(name, unicode)
1137         d = node.get_child_at_path(name)
1138         d.addCallback(lambda node: node.download_to_data())
1139         def _check(contents):
1140             self.failUnlessEqual(contents, expected_contents)
1141         d.addCallback(_check)
1142         return d
1143
1144     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1145         assert isinstance(name, unicode)
1146         d = node.get_child_at_path(name)
1147         d.addCallback(lambda node: node.download_best_version())
1148         def _check(contents):
1149             self.failUnlessEqual(contents, expected_contents)
1150         d.addCallback(_check)
1151         return d
1152
1153     def failUnlessChildURIIs(self, node, name, expected_uri):
1154         assert isinstance(name, unicode)
1155         d = node.get_child_at_path(name)
1156         def _check(child):
1157             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1158         d.addCallback(_check)
1159         return d
1160
1161     def failUnlessURIMatchesChild(self, got_uri, node, name):
1162         assert isinstance(name, unicode)
1163         d = node.get_child_at_path(name)
1164         def _check(child):
1165             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1166         d.addCallback(_check)
1167         return d
1168
1169     def failUnlessCHKURIHasContents(self, got_uri, contents):
1170         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1171
1172     def test_POST_upload(self):
1173         d = self.POST(self.public_url + "/foo", t="upload",
1174                       file=("new.txt", self.NEWFILE_CONTENTS))
1175         fn = self._foo_node
1176         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1177         d.addCallback(lambda res:
1178                       self.failUnlessChildContentsAre(fn, u"new.txt",
1179                                                       self.NEWFILE_CONTENTS))
1180         return d
1181
1182     def test_POST_upload_unicode(self):
1183         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1184         d = self.POST(self.public_url + "/foo", t="upload",
1185                       file=(filename, self.NEWFILE_CONTENTS))
1186         fn = self._foo_node
1187         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1188         d.addCallback(lambda res:
1189                       self.failUnlessChildContentsAre(fn, filename,
1190                                                       self.NEWFILE_CONTENTS))
1191         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1192         d.addCallback(lambda res: self.GET(target_url))
1193         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1194                                                             self.NEWFILE_CONTENTS,
1195                                                             contents))
1196         return d
1197
1198     def test_POST_upload_unicode_named(self):
1199         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1200         d = self.POST(self.public_url + "/foo", t="upload",
1201                       name=filename,
1202                       file=("overridden", self.NEWFILE_CONTENTS))
1203         fn = self._foo_node
1204         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1205         d.addCallback(lambda res:
1206                       self.failUnlessChildContentsAre(fn, filename,
1207                                                       self.NEWFILE_CONTENTS))
1208         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1209         d.addCallback(lambda res: self.GET(target_url))
1210         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1211                                                             self.NEWFILE_CONTENTS,
1212                                                             contents))
1213         return d
1214
1215     def test_POST_upload_no_link(self):
1216         d = self.POST("/uri", t="upload",
1217                       file=("new.txt", self.NEWFILE_CONTENTS))
1218         def _check_upload_results(page):
1219             # this should be a page which describes the results of the upload
1220             # that just finished.
1221             self.failUnless("Upload Results:" in page)
1222             self.failUnless("URI:" in page)
1223             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1224             mo = uri_re.search(page)
1225             self.failUnless(mo, page)
1226             new_uri = mo.group(1)
1227             return new_uri
1228         d.addCallback(_check_upload_results)
1229         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1230         return d
1231
1232     def test_POST_upload_no_link_whendone(self):
1233         d = self.POST("/uri", t="upload", when_done="/",
1234                       file=("new.txt", self.NEWFILE_CONTENTS))
1235         d.addBoth(self.shouldRedirect, "/")
1236         return d
1237
1238     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1239         d = defer.maybeDeferred(callable, *args, **kwargs)
1240         def done(res):
1241             if isinstance(res, failure.Failure):
1242                 res.trap(error.PageRedirect)
1243                 statuscode = res.value.status
1244                 target = res.value.location
1245                 return checker(statuscode, target)
1246             self.fail("%s: callable was supposed to redirect, not return '%s'"
1247                       % (which, res))
1248         d.addBoth(done)
1249         return d
1250
1251     def test_POST_upload_no_link_whendone_results(self):
1252         def check(statuscode, target):
1253             self.failUnlessEqual(statuscode, str(http.FOUND))
1254             self.failUnless(target.startswith(self.webish_url), target)
1255             return client.getPage(target, method="GET")
1256         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1257                                  check,
1258                                  self.POST, "/uri", t="upload",
1259                                  when_done="/uri/%(uri)s",
1260                                  file=("new.txt", self.NEWFILE_CONTENTS))
1261         d.addCallback(lambda res:
1262                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1263         return d
1264
1265     def test_POST_upload_no_link_mutable(self):
1266         d = self.POST("/uri", t="upload", mutable="true",
1267                       file=("new.txt", self.NEWFILE_CONTENTS))
1268         def _check(new_uri):
1269             new_uri = new_uri.strip()
1270             self.new_uri = new_uri
1271             u = IURI(new_uri)
1272             self.failUnless(IMutableFileURI.providedBy(u))
1273             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1274             n = self.s.create_node_from_uri(new_uri)
1275             return n.download_best_version()
1276         d.addCallback(_check)
1277         def _check2(data):
1278             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1279             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1280         d.addCallback(_check2)
1281         def _check3(data):
1282             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1283             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1284         d.addCallback(_check3)
1285         def _check4(data):
1286             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1287         d.addCallback(_check4)
1288         return d
1289
1290     def test_POST_upload_no_link_mutable_toobig(self):
1291         d = self.shouldFail2(error.Error,
1292                              "test_POST_upload_no_link_mutable_toobig",
1293                              "413 Request Entity Too Large",
1294                              "SDMF is limited to one segment, and 10001 > 10000",
1295                              self.POST,
1296                              "/uri", t="upload", mutable="true",
1297                              file=("new.txt",
1298                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1299         return d
1300
1301     def test_POST_upload_mutable(self):
1302         # this creates a mutable file
1303         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1304                       file=("new.txt", self.NEWFILE_CONTENTS))
1305         fn = self._foo_node
1306         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1307         d.addCallback(lambda res:
1308                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1309                                                              self.NEWFILE_CONTENTS))
1310         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1311         def _got(newnode):
1312             self.failUnless(IMutableFileNode.providedBy(newnode))
1313             self.failUnless(newnode.is_mutable())
1314             self.failIf(newnode.is_readonly())
1315             self._mutable_node = newnode
1316             self._mutable_uri = newnode.get_uri()
1317         d.addCallback(_got)
1318
1319         # now upload it again and make sure that the URI doesn't change
1320         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1321         d.addCallback(lambda res:
1322                       self.POST(self.public_url + "/foo", t="upload",
1323                                 mutable="true",
1324                                 file=("new.txt", NEWER_CONTENTS)))
1325         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1326         d.addCallback(lambda res:
1327                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1328                                                              NEWER_CONTENTS))
1329         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1330         def _got2(newnode):
1331             self.failUnless(IMutableFileNode.providedBy(newnode))
1332             self.failUnless(newnode.is_mutable())
1333             self.failIf(newnode.is_readonly())
1334             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1335         d.addCallback(_got2)
1336
1337         # upload a second time, using PUT instead of POST
1338         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1339         d.addCallback(lambda res:
1340                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1341         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1342         d.addCallback(lambda res:
1343                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1344                                                              NEW2_CONTENTS))
1345
1346         # finally list the directory, since mutable files are displayed
1347         # slightly differently
1348
1349         d.addCallback(lambda res:
1350                       self.GET(self.public_url + "/foo/",
1351                                followRedirect=True))
1352         def _check_page(res):
1353             # TODO: assert more about the contents
1354             self.failUnless("SSK" in res)
1355             return res
1356         d.addCallback(_check_page)
1357
1358         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1359         def _got3(newnode):
1360             self.failUnless(IMutableFileNode.providedBy(newnode))
1361             self.failUnless(newnode.is_mutable())
1362             self.failIf(newnode.is_readonly())
1363             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1364         d.addCallback(_got3)
1365
1366         # look at the JSON form of the enclosing directory
1367         d.addCallback(lambda res:
1368                       self.GET(self.public_url + "/foo/?t=json",
1369                                followRedirect=True))
1370         def _check_page_json(res):
1371             parsed = simplejson.loads(res)
1372             self.failUnlessEqual(parsed[0], "dirnode")
1373             children = dict( [(unicode(name),value)
1374                               for (name,value)
1375                               in parsed[1]["children"].iteritems()] )
1376             self.failUnless("new.txt" in children)
1377             new_json = children["new.txt"]
1378             self.failUnlessEqual(new_json[0], "filenode")
1379             self.failUnless(new_json[1]["mutable"])
1380             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1381             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1382             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1383         d.addCallback(_check_page_json)
1384
1385         # and the JSON form of the file
1386         d.addCallback(lambda res:
1387                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1388         def _check_file_json(res):
1389             parsed = simplejson.loads(res)
1390             self.failUnlessEqual(parsed[0], "filenode")
1391             self.failUnless(parsed[1]["mutable"])
1392             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1393             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1394             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1395         d.addCallback(_check_file_json)
1396
1397         # and look at t=uri and t=readonly-uri
1398         d.addCallback(lambda res:
1399                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1400         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1401         d.addCallback(lambda res:
1402                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1403         def _check_ro_uri(res):
1404             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1405             self.failUnlessEqual(res, ro_uri)
1406         d.addCallback(_check_ro_uri)
1407
1408         # make sure we can get to it from /uri/URI
1409         d.addCallback(lambda res:
1410                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1411         d.addCallback(lambda res:
1412                       self.failUnlessEqual(res, NEW2_CONTENTS))
1413
1414         # and that HEAD computes the size correctly
1415         d.addCallback(lambda res:
1416                       self.HEAD(self.public_url + "/foo/new.txt",
1417                                 return_response=True))
1418         def _got_headers((res, status, headers)):
1419             self.failUnlessEqual(res, "")
1420             self.failUnlessEqual(headers["content-length"][0],
1421                                  str(len(NEW2_CONTENTS)))
1422             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1423         d.addCallback(_got_headers)
1424
1425         # make sure that size errors are displayed correctly for overwrite
1426         d.addCallback(lambda res:
1427                       self.shouldFail2(error.Error,
1428                                        "test_POST_upload_mutable-toobig",
1429                                        "413 Request Entity Too Large",
1430                                        "SDMF is limited to one segment, and 10001 > 10000",
1431                                        self.POST,
1432                                        self.public_url + "/foo", t="upload",
1433                                        mutable="true",
1434                                        file=("new.txt",
1435                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1436                                        ))
1437
1438         d.addErrback(self.dump_error)
1439         return d
1440
1441     def test_POST_upload_mutable_toobig(self):
1442         d = self.shouldFail2(error.Error,
1443                              "test_POST_upload_no_link_mutable_toobig",
1444                              "413 Request Entity Too Large",
1445                              "SDMF is limited to one segment, and 10001 > 10000",
1446                              self.POST,
1447                              self.public_url + "/foo",
1448                              t="upload", mutable="true",
1449                              file=("new.txt",
1450                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1451         return d
1452
1453     def dump_error(self, f):
1454         # if the web server returns an error code (like 400 Bad Request),
1455         # web.client.getPage puts the HTTP response body into the .response
1456         # attribute of the exception object that it gives back. It does not
1457         # appear in the Failure's repr(), so the ERROR that trial displays
1458         # will be rather terse and unhelpful. addErrback this method to the
1459         # end of your chain to get more information out of these errors.
1460         if f.check(error.Error):
1461             print "web.error.Error:"
1462             print f
1463             print f.value.response
1464         return f
1465
1466     def test_POST_upload_replace(self):
1467         d = self.POST(self.public_url + "/foo", t="upload",
1468                       file=("bar.txt", self.NEWFILE_CONTENTS))
1469         fn = self._foo_node
1470         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1471         d.addCallback(lambda res:
1472                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1473                                                       self.NEWFILE_CONTENTS))
1474         return d
1475
1476     def test_POST_upload_no_replace_ok(self):
1477         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1478                       file=("new.txt", self.NEWFILE_CONTENTS))
1479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1480         d.addCallback(lambda res: self.failUnlessEqual(res,
1481                                                        self.NEWFILE_CONTENTS))
1482         return d
1483
1484     def test_POST_upload_no_replace_queryarg(self):
1485         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1486                       file=("bar.txt", self.NEWFILE_CONTENTS))
1487         d.addBoth(self.shouldFail, error.Error,
1488                   "POST_upload_no_replace_queryarg",
1489                   "409 Conflict",
1490                   "There was already a child by that name, and you asked me "
1491                   "to not replace it")
1492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1493         d.addCallback(self.failUnlessIsBarDotTxt)
1494         return d
1495
1496     def test_POST_upload_no_replace_field(self):
1497         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1498                       file=("bar.txt", self.NEWFILE_CONTENTS))
1499         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1500                   "409 Conflict",
1501                   "There was already a child by that name, and you asked me "
1502                   "to not replace it")
1503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1504         d.addCallback(self.failUnlessIsBarDotTxt)
1505         return d
1506
1507     def test_POST_upload_whendone(self):
1508         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1509                       file=("new.txt", self.NEWFILE_CONTENTS))
1510         d.addBoth(self.shouldRedirect, "/THERE")
1511         fn = self._foo_node
1512         d.addCallback(lambda res:
1513                       self.failUnlessChildContentsAre(fn, u"new.txt",
1514                                                       self.NEWFILE_CONTENTS))
1515         return d
1516
1517     def test_POST_upload_named(self):
1518         fn = self._foo_node
1519         d = self.POST(self.public_url + "/foo", t="upload",
1520                       name="new.txt", file=self.NEWFILE_CONTENTS)
1521         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1522         d.addCallback(lambda res:
1523                       self.failUnlessChildContentsAre(fn, u"new.txt",
1524                                                       self.NEWFILE_CONTENTS))
1525         return d
1526
1527     def test_POST_upload_named_badfilename(self):
1528         d = self.POST(self.public_url + "/foo", t="upload",
1529                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1530         d.addBoth(self.shouldFail, error.Error,
1531                   "test_POST_upload_named_badfilename",
1532                   "400 Bad Request",
1533                   "name= may not contain a slash",
1534                   )
1535         # make sure that nothing was added
1536         d.addCallback(lambda res:
1537                       self.failUnlessNodeKeysAre(self._foo_node,
1538                                                  [u"bar.txt", u"blockingfile",
1539                                                   u"empty", u"n\u00fc.txt",
1540                                                   u"sub"]))
1541         return d
1542
1543     def test_POST_FILEURL_check(self):
1544         bar_url = self.public_url + "/foo/bar.txt"
1545         d = self.POST(bar_url, t="check")
1546         def _check(res):
1547             self.failUnless("Healthy!" in res)
1548         d.addCallback(_check)
1549         redir_url = "http://allmydata.org/TARGET"
1550         def _check2(statuscode, target):
1551             self.failUnlessEqual(statuscode, str(http.FOUND))
1552             self.failUnlessEqual(target, redir_url)
1553         d.addCallback(lambda res:
1554                       self.shouldRedirect2("test_POST_FILEURL_check",
1555                                            _check2,
1556                                            self.POST, bar_url,
1557                                            t="check",
1558                                            when_done=redir_url))
1559         d.addCallback(lambda res:
1560                       self.POST(bar_url, t="check", return_to=redir_url))
1561         def _check3(res):
1562             self.failUnless("Healthy!" in res)
1563             self.failUnless("Return to parent directory" in res)
1564             self.failUnless(redir_url in res)
1565         d.addCallback(_check3)
1566
1567         d.addCallback(lambda res:
1568                       self.POST(bar_url, t="check", output="JSON"))
1569         def _check_json(res):
1570             data = simplejson.loads(res)
1571             self.failUnless("storage-index" in data)
1572             self.failUnless(data["results"]["healthy"])
1573         d.addCallback(_check_json)
1574
1575         return d
1576
1577     def test_POST_FILEURL_check_and_repair(self):
1578         bar_url = self.public_url + "/foo/bar.txt"
1579         d = self.POST(bar_url, t="check", repair="true")
1580         def _check(res):
1581             self.failUnless("Healthy!" in res)
1582         d.addCallback(_check)
1583         redir_url = "http://allmydata.org/TARGET"
1584         def _check2(statuscode, target):
1585             self.failUnlessEqual(statuscode, str(http.FOUND))
1586             self.failUnlessEqual(target, redir_url)
1587         d.addCallback(lambda res:
1588                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1589                                            _check2,
1590                                            self.POST, bar_url,
1591                                            t="check", repair="true",
1592                                            when_done=redir_url))
1593         d.addCallback(lambda res:
1594                       self.POST(bar_url, t="check", return_to=redir_url))
1595         def _check3(res):
1596             self.failUnless("Healthy!" in res)
1597             self.failUnless("Return to parent directory" in res)
1598             self.failUnless(redir_url in res)
1599         d.addCallback(_check3)
1600         return d
1601
1602     def test_POST_DIRURL_check(self):
1603         foo_url = self.public_url + "/foo/"
1604         d = self.POST(foo_url, t="check")
1605         def _check(res):
1606             self.failUnless("Healthy!" in res)
1607         d.addCallback(_check)
1608         redir_url = "http://allmydata.org/TARGET"
1609         def _check2(statuscode, target):
1610             self.failUnlessEqual(statuscode, str(http.FOUND))
1611             self.failUnlessEqual(target, redir_url)
1612         d.addCallback(lambda res:
1613                       self.shouldRedirect2("test_POST_DIRURL_check",
1614                                            _check2,
1615                                            self.POST, foo_url,
1616                                            t="check",
1617                                            when_done=redir_url))
1618         d.addCallback(lambda res:
1619                       self.POST(foo_url, t="check", return_to=redir_url))
1620         def _check3(res):
1621             self.failUnless("Healthy!" in res)
1622             self.failUnless("Return to parent directory" in res)
1623             self.failUnless(redir_url in res)
1624         d.addCallback(_check3)
1625
1626         d.addCallback(lambda res:
1627                       self.POST(foo_url, t="check", output="JSON"))
1628         def _check_json(res):
1629             data = simplejson.loads(res)
1630             self.failUnless("storage-index" in data)
1631             self.failUnless(data["results"]["healthy"])
1632         d.addCallback(_check_json)
1633
1634         return d
1635
1636     def test_POST_DIRURL_check_and_repair(self):
1637         foo_url = self.public_url + "/foo/"
1638         d = self.POST(foo_url, t="check", repair="true")
1639         def _check(res):
1640             self.failUnless("Healthy!" in res)
1641         d.addCallback(_check)
1642         redir_url = "http://allmydata.org/TARGET"
1643         def _check2(statuscode, target):
1644             self.failUnlessEqual(statuscode, str(http.FOUND))
1645             self.failUnlessEqual(target, redir_url)
1646         d.addCallback(lambda res:
1647                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1648                                            _check2,
1649                                            self.POST, foo_url,
1650                                            t="check", repair="true",
1651                                            when_done=redir_url))
1652         d.addCallback(lambda res:
1653                       self.POST(foo_url, t="check", return_to=redir_url))
1654         def _check3(res):
1655             self.failUnless("Healthy!" in res)
1656             self.failUnless("Return to parent directory" in res)
1657             self.failUnless(redir_url in res)
1658         d.addCallback(_check3)
1659         return d
1660
1661     def wait_for_operation(self, ignored, ophandle):
1662         url = "/operations/" + ophandle
1663         url += "?t=status&output=JSON"
1664         d = self.GET(url)
1665         def _got(res):
1666             data = simplejson.loads(res)
1667             if not data["finished"]:
1668                 d = self.stall(delay=1.0)
1669                 d.addCallback(self.wait_for_operation, ophandle)
1670                 return d
1671             return data
1672         d.addCallback(_got)
1673         return d
1674
1675     def get_operation_results(self, ignored, ophandle, output=None):
1676         url = "/operations/" + ophandle
1677         url += "?t=status"
1678         if output:
1679             url += "&output=" + output
1680         d = self.GET(url)
1681         def _got(res):
1682             if output and output.lower() == "json":
1683                 return simplejson.loads(res)
1684             return res
1685         d.addCallback(_got)
1686         return d
1687
1688     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1689         d = self.shouldFail2(error.Error,
1690                              "test_POST_DIRURL_deepcheck_no_ophandle",
1691                              "400 Bad Request",
1692                              "slow operation requires ophandle=",
1693                              self.POST, self.public_url, t="start-deep-check")
1694         return d
1695
1696     def test_POST_DIRURL_deepcheck(self):
1697         def _check_redirect(statuscode, target):
1698             self.failUnlessEqual(statuscode, str(http.FOUND))
1699             self.failUnless(target.endswith("/operations/123"))
1700         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1701                                  self.POST, self.public_url,
1702                                  t="start-deep-check", ophandle="123")
1703         d.addCallback(self.wait_for_operation, "123")
1704         def _check_json(data):
1705             self.failUnlessEqual(data["finished"], True)
1706             self.failUnlessEqual(data["count-objects-checked"], 8)
1707             self.failUnlessEqual(data["count-objects-healthy"], 8)
1708         d.addCallback(_check_json)
1709         d.addCallback(self.get_operation_results, "123", "html")
1710         def _check_html(res):
1711             self.failUnless("Objects Checked: <span>8</span>" in res)
1712             self.failUnless("Objects Healthy: <span>8</span>" in res)
1713         d.addCallback(_check_html)
1714
1715         d.addCallback(lambda res:
1716                       self.GET("/operations/123/"))
1717         d.addCallback(_check_html) # should be the same as without the slash
1718
1719         d.addCallback(lambda res:
1720                       self.shouldFail2(error.Error, "one", "404 Not Found",
1721                                        "No detailed results for SI bogus",
1722                                        self.GET, "/operations/123/bogus"))
1723
1724         foo_si = self._foo_node.get_storage_index()
1725         foo_si_s = base32.b2a(foo_si)
1726         d.addCallback(lambda res:
1727                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1728         def _check_foo_json(res):
1729             data = simplejson.loads(res)
1730             self.failUnlessEqual(data["storage-index"], foo_si_s)
1731             self.failUnless(data["results"]["healthy"])
1732         d.addCallback(_check_foo_json)
1733         return d
1734
1735     def test_POST_DIRURL_deepcheck_and_repair(self):
1736         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1737                       ophandle="124", output="json", followRedirect=True)
1738         d.addCallback(self.wait_for_operation, "124")
1739         def _check_json(data):
1740             self.failUnlessEqual(data["finished"], True)
1741             self.failUnlessEqual(data["count-objects-checked"], 8)
1742             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1743             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1744             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1745             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1746             self.failUnlessEqual(data["count-repairs-successful"], 0)
1747             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1748             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1749             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1750             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1751         d.addCallback(_check_json)
1752         d.addCallback(self.get_operation_results, "124", "html")
1753         def _check_html(res):
1754             self.failUnless("Objects Checked: <span>8</span>" in res)
1755
1756             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1757             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1758             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1759
1760             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1761             self.failUnless("Repairs Successful: <span>0</span>" in res)
1762             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1763
1764             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1765             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1766             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1767         d.addCallback(_check_html)
1768         return d
1769
1770     def test_POST_FILEURL_bad_t(self):
1771         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1772                              "POST to file: bad t=bogus",
1773                              self.POST, self.public_url + "/foo/bar.txt",
1774                              t="bogus")
1775         return d
1776
1777     def test_POST_mkdir(self): # return value?
1778         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1779         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1780         d.addCallback(self.failUnlessNodeKeysAre, [])
1781         return d
1782
1783     def test_POST_mkdir_2(self):
1784         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1785         d.addCallback(lambda res:
1786                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1787         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1788         d.addCallback(self.failUnlessNodeKeysAre, [])
1789         return d
1790
1791     def test_POST_mkdirs_2(self):
1792         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1793         d.addCallback(lambda res:
1794                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1795         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1796         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1797         d.addCallback(self.failUnlessNodeKeysAre, [])
1798         return d
1799
1800     def test_POST_mkdir_no_parentdir_noredirect(self):
1801         d = self.POST("/uri?t=mkdir")
1802         def _after_mkdir(res):
1803             uri.NewDirectoryURI.init_from_string(res)
1804         d.addCallback(_after_mkdir)
1805         return d
1806
1807     def test_POST_mkdir_no_parentdir_redirect(self):
1808         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1809         d.addBoth(self.shouldRedirect, None, statuscode='303')
1810         def _check_target(target):
1811             target = urllib.unquote(target)
1812             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1813         d.addCallback(_check_target)
1814         return d
1815
1816     def test_POST_noparent_bad(self):
1817         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1818                                   "/uri accepts only PUT, PUT?t=mkdir, "
1819                                   "POST?t=upload, and POST?t=mkdir",
1820                                   self.POST, "/uri?t=bogus")
1821         return d
1822
1823     def test_welcome_page_mkdir_button(self):
1824         # Fetch the welcome page.
1825         d = self.GET("/")
1826         def _after_get_welcome_page(res):
1827             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1828             mo = MKDIR_BUTTON_RE.search(res)
1829             formaction = mo.group(1)
1830             formt = mo.group(2)
1831             formaname = mo.group(3)
1832             formavalue = mo.group(4)
1833             return (formaction, formt, formaname, formavalue)
1834         d.addCallback(_after_get_welcome_page)
1835         def _after_parse_form(res):
1836             (formaction, formt, formaname, formavalue) = res
1837             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1838         d.addCallback(_after_parse_form)
1839         d.addBoth(self.shouldRedirect, None, statuscode='303')
1840         return d
1841
1842     def test_POST_mkdir_replace(self): # return value?
1843         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1844         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1845         d.addCallback(self.failUnlessNodeKeysAre, [])
1846         return d
1847
1848     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1849         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1850         d.addBoth(self.shouldFail, error.Error,
1851                   "POST_mkdir_no_replace_queryarg",
1852                   "409 Conflict",
1853                   "There was already a child by that name, and you asked me "
1854                   "to not replace it")
1855         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1856         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1857         return d
1858
1859     def test_POST_mkdir_no_replace_field(self): # return value?
1860         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1861                       replace="false")
1862         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1863                   "409 Conflict",
1864                   "There was already a child by that name, and you asked me "
1865                   "to not replace it")
1866         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1867         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1868         return d
1869
1870     def test_POST_mkdir_whendone_field(self):
1871         d = self.POST(self.public_url + "/foo",
1872                       t="mkdir", name="newdir", when_done="/THERE")
1873         d.addBoth(self.shouldRedirect, "/THERE")
1874         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875         d.addCallback(self.failUnlessNodeKeysAre, [])
1876         return d
1877
1878     def test_POST_mkdir_whendone_queryarg(self):
1879         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1880                       t="mkdir", name="newdir")
1881         d.addBoth(self.shouldRedirect, "/THERE")
1882         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1883         d.addCallback(self.failUnlessNodeKeysAre, [])
1884         return d
1885
1886     def test_POST_bad_t(self):
1887         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1888                              "POST to a directory with bad t=BOGUS",
1889                              self.POST, self.public_url + "/foo", t="BOGUS")
1890         return d
1891
1892     def test_POST_set_children(self):
1893         contents9, n9, newuri9 = self.makefile(9)
1894         contents10, n10, newuri10 = self.makefile(10)
1895         contents11, n11, newuri11 = self.makefile(11)
1896
1897         reqbody = """{
1898                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1899                                                 "size": 0,
1900                                                 "metadata": {
1901                                                   "ctime": 1002777696.7564139,
1902                                                   "mtime": 1002777696.7564139
1903                                                  }
1904                                                } ],
1905                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1906                                                 "size": 1,
1907                                                 "metadata": {
1908                                                   "ctime": 1002777696.7564139,
1909                                                   "mtime": 1002777696.7564139
1910                                                  }
1911                                                } ],
1912                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1913                                                 "size": 2,
1914                                                 "metadata": {
1915                                                   "ctime": 1002777696.7564139,
1916                                                   "mtime": 1002777696.7564139
1917                                                  }
1918                                                } ]
1919                     }""" % (newuri9, newuri10, newuri11)
1920
1921         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1922
1923         d = client.getPage(url, method="POST", postdata=reqbody)
1924         def _then(res):
1925             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1926             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1927             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1928
1929         d.addCallback(_then)
1930         d.addErrback(self.dump_error)
1931         return d
1932
1933     def test_POST_put_uri(self):
1934         contents, n, newuri = self.makefile(8)
1935         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1936         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1937         d.addCallback(lambda res:
1938                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1939                                                       contents))
1940         return d
1941
1942     def test_POST_put_uri_replace(self):
1943         contents, n, newuri = self.makefile(8)
1944         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1945         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1946         d.addCallback(lambda res:
1947                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1948                                                       contents))
1949         return d
1950
1951     def test_POST_put_uri_no_replace_queryarg(self):
1952         contents, n, newuri = self.makefile(8)
1953         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1954                       name="bar.txt", uri=newuri)
1955         d.addBoth(self.shouldFail, error.Error,
1956                   "POST_put_uri_no_replace_queryarg",
1957                   "409 Conflict",
1958                   "There was already a child by that name, and you asked me "
1959                   "to not replace it")
1960         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1961         d.addCallback(self.failUnlessIsBarDotTxt)
1962         return d
1963
1964     def test_POST_put_uri_no_replace_field(self):
1965         contents, n, newuri = self.makefile(8)
1966         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1967                       name="bar.txt", uri=newuri)
1968         d.addBoth(self.shouldFail, error.Error,
1969                   "POST_put_uri_no_replace_field",
1970                   "409 Conflict",
1971                   "There was already a child by that name, and you asked me "
1972                   "to not replace it")
1973         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1974         d.addCallback(self.failUnlessIsBarDotTxt)
1975         return d
1976
1977     def test_POST_delete(self):
1978         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1979         d.addCallback(lambda res: self._foo_node.list())
1980         def _check(children):
1981             self.failIf(u"bar.txt" in children)
1982         d.addCallback(_check)
1983         return d
1984
1985     def test_POST_rename_file(self):
1986         d = self.POST(self.public_url + "/foo", t="rename",
1987                       from_name="bar.txt", to_name='wibble.txt')
1988         d.addCallback(lambda res:
1989                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1990         d.addCallback(lambda res:
1991                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1992         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1993         d.addCallback(self.failUnlessIsBarDotTxt)
1994         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1995         d.addCallback(self.failUnlessIsBarJSON)
1996         return d
1997
1998     def test_POST_rename_file_redundant(self):
1999         d = self.POST(self.public_url + "/foo", t="rename",
2000                       from_name="bar.txt", to_name='bar.txt')
2001         d.addCallback(lambda res:
2002                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2003         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2004         d.addCallback(self.failUnlessIsBarDotTxt)
2005         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2006         d.addCallback(self.failUnlessIsBarJSON)
2007         return d
2008
2009     def test_POST_rename_file_replace(self):
2010         # rename a file and replace a directory with it
2011         d = self.POST(self.public_url + "/foo", t="rename",
2012                       from_name="bar.txt", to_name='empty')
2013         d.addCallback(lambda res:
2014                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2015         d.addCallback(lambda res:
2016                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2017         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2018         d.addCallback(self.failUnlessIsBarDotTxt)
2019         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2020         d.addCallback(self.failUnlessIsBarJSON)
2021         return d
2022
2023     def test_POST_rename_file_no_replace_queryarg(self):
2024         # rename a file and replace a directory with it
2025         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2026                       from_name="bar.txt", to_name='empty')
2027         d.addBoth(self.shouldFail, error.Error,
2028                   "POST_rename_file_no_replace_queryarg",
2029                   "409 Conflict",
2030                   "There was already a child by that name, and you asked me "
2031                   "to not replace it")
2032         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2033         d.addCallback(self.failUnlessIsEmptyJSON)
2034         return d
2035
2036     def test_POST_rename_file_no_replace_field(self):
2037         # rename a file and replace a directory with it
2038         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2039                       from_name="bar.txt", to_name='empty')
2040         d.addBoth(self.shouldFail, error.Error,
2041                   "POST_rename_file_no_replace_field",
2042                   "409 Conflict",
2043                   "There was already a child by that name, and you asked me "
2044                   "to not replace it")
2045         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2046         d.addCallback(self.failUnlessIsEmptyJSON)
2047         return d
2048
2049     def failUnlessIsEmptyJSON(self, res):
2050         data = simplejson.loads(res)
2051         self.failUnlessEqual(data[0], "dirnode", data)
2052         self.failUnlessEqual(len(data[1]["children"]), 0)
2053
2054     def test_POST_rename_file_slash_fail(self):
2055         d = self.POST(self.public_url + "/foo", t="rename",
2056                       from_name="bar.txt", to_name='kirk/spock.txt')
2057         d.addBoth(self.shouldFail, error.Error,
2058                   "test_POST_rename_file_slash_fail",
2059                   "400 Bad Request",
2060                   "to_name= may not contain a slash",
2061                   )
2062         d.addCallback(lambda res:
2063                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2064         return d
2065
2066     def test_POST_rename_dir(self):
2067         d = self.POST(self.public_url, t="rename",
2068                       from_name="foo", to_name='plunk')
2069         d.addCallback(lambda res:
2070                       self.failIfNodeHasChild(self.public_root, u"foo"))
2071         d.addCallback(lambda res:
2072                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2073         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2074         d.addCallback(self.failUnlessIsFooJSON)
2075         return d
2076
2077     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2078         """ If target is not None then the redirection has to go to target.  If
2079         statuscode is not None then the redirection has to be accomplished with
2080         that HTTP status code."""
2081         if not isinstance(res, failure.Failure):
2082             to_where = (target is None) and "somewhere" or ("to " + target)
2083             self.fail("%s: we were expecting to get redirected %s, not get an"
2084                       " actual page: %s" % (which, to_where, res))
2085         res.trap(error.PageRedirect)
2086         if statuscode is not None:
2087             self.failUnlessEqual(res.value.status, statuscode,
2088                                  "%s: not a redirect" % which)
2089         if target is not None:
2090             # the PageRedirect does not seem to capture the uri= query arg
2091             # properly, so we can't check for it.
2092             realtarget = self.webish_url + target
2093             self.failUnlessEqual(res.value.location, realtarget,
2094                                  "%s: wrong target" % which)
2095         return res.value.location
2096
2097     def test_GET_URI_form(self):
2098         base = "/uri?uri=%s" % self._bar_txt_uri
2099         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2100         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2101         d = self.GET(base)
2102         d.addBoth(self.shouldRedirect, targetbase)
2103         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2104         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2105         d.addCallback(lambda res: self.GET(base+"&t=json"))
2106         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2107         d.addCallback(self.log, "about to get file by uri")
2108         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2109         d.addCallback(self.failUnlessIsBarDotTxt)
2110         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2111         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2112                                            followRedirect=True))
2113         d.addCallback(self.failUnlessIsFooJSON)
2114         d.addCallback(self.log, "got dir by uri")
2115
2116         return d
2117
2118     def test_GET_URI_form_bad(self):
2119         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2120                              "400 Bad Request", "GET /uri requires uri=",
2121                              self.GET, "/uri")
2122         return d
2123
2124     def test_GET_rename_form(self):
2125         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2126                      followRedirect=True)
2127         def _check(res):
2128             self.failUnless('name="when_done" value="."' in res, res)
2129             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2130         d.addCallback(_check)
2131         return d
2132
2133     def log(self, res, msg):
2134         #print "MSG: %s  RES: %s" % (msg, res)
2135         log.msg(msg)
2136         return res
2137
2138     def test_GET_URI_URL(self):
2139         base = "/uri/%s" % self._bar_txt_uri
2140         d = self.GET(base)
2141         d.addCallback(self.failUnlessIsBarDotTxt)
2142         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2143         d.addCallback(self.failUnlessIsBarDotTxt)
2144         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2145         d.addCallback(self.failUnlessIsBarDotTxt)
2146         return d
2147
2148     def test_GET_URI_URL_dir(self):
2149         base = "/uri/%s?t=json" % self._foo_uri
2150         d = self.GET(base)
2151         d.addCallback(self.failUnlessIsFooJSON)
2152         return d
2153
2154     def test_GET_URI_URL_missing(self):
2155         base = "/uri/%s" % self._bad_file_uri
2156         d = self.GET(base)
2157         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2158                   http.GONE, response_substring="NotEnoughSharesError")
2159         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2160         # here? we must arrange for a download to fail after target.open()
2161         # has been called, and then inspect the response to see that it is
2162         # shorter than we expected.
2163         return d
2164
2165     def test_PUT_NEWFILEURL_uri(self):
2166         contents, n, new_uri = self.makefile(8)
2167         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2168         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2169         d.addCallback(lambda res:
2170                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2171                                                       contents))
2172         return d
2173
2174     def test_PUT_NEWFILEURL_uri_replace(self):
2175         contents, n, new_uri = self.makefile(8)
2176         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2177         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2178         d.addCallback(lambda res:
2179                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2180                                                       contents))
2181         return d
2182
2183     def test_PUT_NEWFILEURL_uri_no_replace(self):
2184         contents, n, new_uri = self.makefile(8)
2185         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2186         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2187                   "409 Conflict",
2188                   "There was already a child by that name, and you asked me "
2189                   "to not replace it")
2190         return d
2191
2192     def test_PUT_NEWFILE_URI(self):
2193         file_contents = "New file contents here\n"
2194         d = self.PUT("/uri", file_contents)
2195         def _check(uri):
2196             self.failUnless(uri in FakeCHKFileNode.all_contents)
2197             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2198                                  file_contents)
2199             return self.GET("/uri/%s" % uri)
2200         d.addCallback(_check)
2201         def _check2(res):
2202             self.failUnlessEqual(res, file_contents)
2203         d.addCallback(_check2)
2204         return d
2205
2206     def test_PUT_NEWFILE_URI_only_PUT(self):
2207         d = self.PUT("/uri?t=bogus", "")
2208         d.addBoth(self.shouldFail, error.Error,
2209                   "PUT_NEWFILE_URI_only_PUT",
2210                   "400 Bad Request",
2211                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2212         return d
2213
2214     def test_PUT_NEWFILE_URI_mutable(self):
2215         file_contents = "New file contents here\n"
2216         d = self.PUT("/uri?mutable=true", file_contents)
2217         def _check_mutable(uri):
2218             uri = uri.strip()
2219             u = IURI(uri)
2220             self.failUnless(IMutableFileURI.providedBy(u))
2221             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2222             n = self.s.create_node_from_uri(uri)
2223             return n.download_best_version()
2224         d.addCallback(_check_mutable)
2225         def _check2_mutable(data):
2226             self.failUnlessEqual(data, file_contents)
2227         d.addCallback(_check2_mutable)
2228         return d
2229
2230         def _check(uri):
2231             self.failUnless(uri in FakeCHKFileNode.all_contents)
2232             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2233                                  file_contents)
2234             return self.GET("/uri/%s" % uri)
2235         d.addCallback(_check)
2236         def _check2(res):
2237             self.failUnlessEqual(res, file_contents)
2238         d.addCallback(_check2)
2239         return d
2240
2241     def test_PUT_mkdir(self):
2242         d = self.PUT("/uri?t=mkdir", "")
2243         def _check(uri):
2244             n = self.s.create_node_from_uri(uri.strip())
2245             d2 = self.failUnlessNodeKeysAre(n, [])
2246             d2.addCallback(lambda res:
2247                            self.GET("/uri/%s?t=json" % uri))
2248             return d2
2249         d.addCallback(_check)
2250         d.addCallback(self.failUnlessIsEmptyJSON)
2251         return d
2252
2253     def test_POST_check(self):
2254         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2255         def _done(res):
2256             # this returns a string form of the results, which are probably
2257             # None since we're using fake filenodes.
2258             # TODO: verify that the check actually happened, by changing
2259             # FakeCHKFileNode to count how many times .check() has been
2260             # called.
2261             pass
2262         d.addCallback(_done)
2263         return d
2264
2265     def test_bad_method(self):
2266         url = self.webish_url + self.public_url + "/foo/bar.txt"
2267         d = self.shouldHTTPError2("test_bad_method",
2268                                   501, "Not Implemented",
2269                                   "I don't know how to treat a BOGUS request.",
2270                                   client.getPage, url, method="BOGUS")
2271         return d
2272
2273     def test_short_url(self):
2274         url = self.webish_url + "/uri"
2275         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2276                                   "I don't know how to treat a DELETE request.",
2277                                   client.getPage, url, method="DELETE")
2278         return d
2279
2280     def test_ophandle_bad(self):
2281         url = self.webish_url + "/operations/bogus?t=status"
2282         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2283                                   "unknown/expired handle 'bogus'",
2284                                   client.getPage, url)
2285         return d
2286
2287     def test_ophandle_cancel(self):
2288         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2289                       followRedirect=True)
2290         d.addCallback(lambda ignored:
2291                       self.GET("/operations/128?t=status&output=JSON"))
2292         def _check1(res):
2293             data = simplejson.loads(res)
2294             self.failUnless("finished" in data, res)
2295             monitor = self.ws.root.child_operations.handles["128"][0]
2296             d = self.POST("/operations/128?t=cancel&output=JSON")
2297             def _check2(res):
2298                 data = simplejson.loads(res)
2299                 self.failUnless("finished" in data, res)
2300                 # t=cancel causes the handle to be forgotten
2301                 self.failUnless(monitor.is_cancelled())
2302             d.addCallback(_check2)
2303             return d
2304         d.addCallback(_check1)
2305         d.addCallback(lambda ignored:
2306                       self.shouldHTTPError2("test_ophandle_cancel",
2307                                             404, "404 Not Found",
2308                                             "unknown/expired handle '128'",
2309                                             self.GET,
2310                                             "/operations/128?t=status&output=JSON"))
2311         return d
2312
2313     def test_ophandle_retainfor(self):
2314         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2315                       followRedirect=True)
2316         d.addCallback(lambda ignored:
2317                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2318         def _check1(res):
2319             data = simplejson.loads(res)
2320             self.failUnless("finished" in data, res)
2321         d.addCallback(_check1)
2322         # the retain-for=0 will cause the handle to be expired very soon
2323         d.addCallback(self.stall, 2.0)
2324         d.addCallback(lambda ignored:
2325                       self.shouldHTTPError2("test_ophandle_retainfor",
2326                                             404, "404 Not Found",
2327                                             "unknown/expired handle '129'",
2328                                             self.GET,
2329                                             "/operations/129?t=status&output=JSON"))
2330         return d
2331
2332     def test_ophandle_release_after_complete(self):
2333         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2334                       followRedirect=True)
2335         d.addCallback(self.wait_for_operation, "130")
2336         d.addCallback(lambda ignored:
2337                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2338         # the release-after-complete=true will cause the handle to be expired
2339         d.addCallback(lambda ignored:
2340                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2341                                             404, "404 Not Found",
2342                                             "unknown/expired handle '130'",
2343                                             self.GET,
2344                                             "/operations/130?t=status&output=JSON"))
2345         return d
2346
2347     def test_incident(self):
2348         d = self.POST("/report_incident", details="eek")
2349         def _done(res):
2350             self.failUnless("Thank you for your report!" in res, res)
2351         d.addCallback(_done)
2352         return d
2353
2354
2355 class Util(unittest.TestCase):
2356     def test_abbreviate_time(self):
2357         self.failUnlessEqual(common.abbreviate_time(None), "")
2358         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2359         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2360         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2361         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2362
2363     def test_abbreviate_rate(self):
2364         self.failUnlessEqual(common.abbreviate_rate(None), "")
2365         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2366         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2367         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2368
2369     def test_abbreviate_size(self):
2370         self.failUnlessEqual(common.abbreviate_size(None), "")
2371         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2372         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2373         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2374         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2375