src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, testutil, base32
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17
  18 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  19 # create a webserver that works against them
  20
  21 class FakeIntroducerClient:
  22     def get_all_connectors(self):
  23         return {}
  24     def get_all_connections_for(self, service_name):
  25         return frozenset()
  26     def get_all_peerids(self):
  27         return frozenset()
  28
  29 class FakeClient(service.MultiService):
  30     nodeid = "fake_nodeid"
  31     nickname = "fake_nickname"
  32     basedir = "fake_basedir"
  33     def get_versions(self):
  34         return {'allmydata': "fake",
  35                 'foolscap': "fake",
  36                 'twisted': "fake",
  37                 'zfec': "fake",
  38                 }
  39     introducer_furl = "None"
  40     introducer_client = FakeIntroducerClient()
  41     _all_upload_status = [upload.UploadStatus()]
  42     _all_download_status = [download.DownloadStatus()]
  43     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  44     _all_publish_statuses = [publish.PublishStatus()]
  45     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  46     convergence = "some random string"
  47
  48     def connected_to_introducer(self):
  49         return False
  50
  51     def get_nickname_for_peerid(self, peerid):
  52         return u"John Doe"
  53
  54     def create_node_from_uri(self, auri):
  55         u = uri.from_string(auri)
  56         if (INewDirectoryURI.providedBy(u)
  57             or IReadonlyNewDirectoryURI.providedBy(u)):
  58             return FakeDirectoryNode(self).init_from_uri(u)
  59         if IFileURI.providedBy(u):
  60             return FakeCHKFileNode(u, self)
  61         assert IMutableFileURI.providedBy(u), u
  62         return FakeMutableFileNode(self).init_from_uri(u)
  63
  64     def create_empty_dirnode(self):
  65         n = FakeDirectoryNode(self)
  66         d = n.create()
  67         d.addCallback(lambda res: n)
  68         return d
  69
  70     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  71     def create_mutable_file(self, contents=""):
  72         n = FakeMutableFileNode(self)
  73         return n.create(contents)
  74
  75     def upload(self, uploadable):
  76         d = uploadable.get_size()
  77         d.addCallback(lambda size: uploadable.read(size))
  78         def _got_data(datav):
  79             data = "".join(datav)
  80             n = create_chk_filenode(self, data)
  81             results = upload.UploadResults()
  82             results.uri = n.get_uri()
  83             return results
  84         d.addCallback(_got_data)
  85         return d
  86
  87     def list_all_upload_statuses(self):
  88         return self._all_upload_status
  89     def list_all_download_statuses(self):
  90         return self._all_download_status
  91     def list_all_mapupdate_statuses(self):
  92         return self._all_mapupdate_statuses
  93     def list_all_publish_statuses(self):
  94         return self._all_publish_statuses
  95     def list_all_retrieve_statuses(self):
  96         return self._all_retrieve_statuses
  97     def list_all_helper_statuses(self):
  98         return []
  99
 100 class HTTPClientHEADFactory(client.HTTPClientFactory):
 101     def __init__(self, *args, **kwargs):
 102         client.HTTPClientFactory.__init__(self, *args, **kwargs)
 103         self.deferred.addCallback(lambda res: self.response_headers)
 104
 105     def noPage(self, reason):
 106         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 107         # exception when the response to a HEAD request had no body (when in
 108         # fact they are defined to never have a body). This was fixed in
 109         # Twisted-8.0 . To work around this, we catch the
 110         # PartialDownloadError and make it disappear.
 111         if (reason.check(client.PartialDownloadError)
 112             and self.method.upper() == "HEAD"):
 113             self.page("")
 114             return
 115         return client.HTTPClientFactory.noPage(self, reason)
 116
 117
 118 class WebMixin(object):
 119     def setUp(self):
 120         self.s = FakeClient()
 121         self.s.startService()
 122         self.ws = s = webish.WebishServer("0")
 123         s.setServiceParent(self.s)
 124         self.webish_port = port = s.listener._port.getHost().port
 125         self.webish_url = "http://localhost:%d" % port
 126
 127         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 128         d = defer.DeferredList(l)
 129         def _then(res):
 130             self.public_root = res[0][1]
 131             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 132             self.public_url = "/uri/" + self.public_root.get_uri()
 133             self.private_root = res[1][1]
 134
 135             foo = res[2][1]
 136             self._foo_node = foo
 137             self._foo_uri = foo.get_uri()
 138             self._foo_readonly_uri = foo.get_readonly_uri()
 139             # NOTE: we ignore the deferred on all set_uri() calls, because we
 140             # know the fake nodes do these synchronously
 141             self.public_root.set_uri(u"foo", foo.get_uri())
 142
 143             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 144             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 145
 146             foo.set_uri(u"empty", res[3][1].get_uri())
 147             sub_uri = res[4][1].get_uri()
 148             self._sub_uri = sub_uri
 149             foo.set_uri(u"sub", sub_uri)
 150             sub = self.s.create_node_from_uri(sub_uri)
 151
 152             _ign, n, blocking_uri = self.makefile(1)
 153             foo.set_uri(u"blockingfile", blocking_uri)
 154
 155             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 156             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 157             # still think of it as an umlaut
 158             foo.set_uri(unicode_filename, self._bar_txt_uri)
 159
 160             _ign, n, baz_file = self.makefile(2)
 161             sub.set_uri(u"baz.txt", baz_file)
 162
 163             _ign, n, self._bad_file_uri = self.makefile(3)
 164             # this uri should not be downloadable
 165             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 166
 167             rodir = res[5][1]
 168             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 169             rodir.set_uri(u"nor", baz_file)
 170
 171             # public/
 172             # public/foo/
 173             # public/foo/bar.txt
 174             # public/foo/blockingfile
 175             # public/foo/empty/
 176             # public/foo/sub/
 177             # public/foo/sub/baz.txt
 178             # public/reedownlee/
 179             # public/reedownlee/nor
 180             self.NEWFILE_CONTENTS = "newfile contents\n"
 181
 182             return foo.get_metadata_for(u"bar.txt")
 183         d.addCallback(_then)
 184         def _got_metadata(metadata):
 185             self._bar_txt_metadata = metadata
 186         d.addCallback(_got_metadata)
 187         return d
 188
 189     def makefile(self, number):
 190         contents = "contents of file %s\n" % number
 191         n = create_chk_filenode(self.s, contents)
 192         return contents, n, n.get_uri()
 193
 194     def tearDown(self):
 195         return self.s.stopService()
 196
 197     def failUnlessIsBarDotTxt(self, res):
 198         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 199
 200     def failUnlessIsBarJSON(self, res):
 201         data = simplejson.loads(res)
 202         self.failUnless(isinstance(data, list))
 203         self.failUnlessEqual(data[0], u"filenode")
 204         self.failUnless(isinstance(data[1], dict))
 205         self.failIf(data[1]["mutable"])
 206         self.failIf("rw_uri" in data[1]) # immutable
 207         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 208         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 209
 210     def failUnlessIsFooJSON(self, res):
 211         data = simplejson.loads(res)
 212         self.failUnless(isinstance(data, list))
 213         self.failUnlessEqual(data[0], "dirnode", res)
 214         self.failUnless(isinstance(data[1], dict))
 215         self.failUnless(data[1]["mutable"])
 216         self.failUnless("rw_uri" in data[1]) # mutable
 217         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 218         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 219
 220         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 221         self.failUnlessEqual(kidnames,
 222                              [u"bar.txt", u"blockingfile", u"empty",
 223                               u"n\u00fc.txt", u"sub"])
 224         kids = dict( [(unicode(name),value)
 225                       for (name,value)
 226                       in data[1]["children"].iteritems()] )
 227         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 228         self.failUnless("metadata" in kids[u"sub"][1])
 229         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 230         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 231         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 232         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 233         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 234         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 235                              self._bar_txt_metadata["ctime"])
 236         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 237                              self._bar_txt_uri)
 238
 239     def GET(self, urlpath, followRedirect=False):
 240         assert not isinstance(urlpath, unicode)
 241         url = self.webish_url + urlpath
 242         return client.getPage(url, method="GET", followRedirect=followRedirect)
 243
 244     def HEAD(self, urlpath):
 245         # this requires some surgery, because twisted.web.client doesn't want
 246         # to give us back the response headers.
 247         factory = HTTPClientHEADFactory(urlpath, method="HEAD")
 248         reactor.connectTCP("localhost", self.webish_port, factory)
 249         return factory.deferred
 250
 251     def PUT(self, urlpath, data):
 252         url = self.webish_url + urlpath
 253         return client.getPage(url, method="PUT", postdata=data)
 254
 255     def DELETE(self, urlpath):
 256         url = self.webish_url + urlpath
 257         return client.getPage(url, method="DELETE")
 258
 259     def POST(self, urlpath, followRedirect=False, **fields):
 260         url = self.webish_url + urlpath
 261         sepbase = "boogabooga"
 262         sep = "--" + sepbase
 263         form = []
 264         form.append(sep)
 265         form.append('Content-Disposition: form-data; name="_charset"')
 266         form.append('')
 267         form.append('UTF-8')
 268         form.append(sep)
 269         for name, value in fields.iteritems():
 270             if isinstance(value, tuple):
 271                 filename, value = value
 272                 form.append('Content-Disposition: form-data; name="%s"; '
 273                             'filename="%s"' % (name, filename.encode("utf-8")))
 274             else:
 275                 form.append('Content-Disposition: form-data; name="%s"' % name)
 276             form.append('')
 277             if isinstance(value, unicode):
 278                 value = value.encode("utf-8")
 279             else:
 280                 value = str(value)
 281             assert isinstance(value, str)
 282             form.append(value)
 283             form.append(sep)
 284         form[-1] += "--"
 285         body = "\r\n".join(form) + "\r\n"
 286         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 287                    }
 288         return client.getPage(url, method="POST", postdata=body,
 289                               headers=headers, followRedirect=followRedirect)
 290
 291     def shouldFail(self, res, expected_failure, which,
 292                    substring=None, response_substring=None):
 293         if isinstance(res, failure.Failure):
 294             res.trap(expected_failure)
 295             if substring:
 296                 self.failUnless(substring in str(res),
 297                                 "substring '%s' not in '%s'"
 298                                 % (substring, str(res)))
 299             if response_substring:
 300                 self.failUnless(response_substring in res.value.response,
 301                                 "response substring '%s' not in '%s'"
 302                                 % (response_substring, res.value.response))
 303         else:
 304             self.fail("%s was supposed to raise %s, not get '%s'" %
 305                       (which, expected_failure, res))
 306
 307     def shouldFail2(self, expected_failure, which, substring,
 308                     response_substring,
 309                     callable, *args, **kwargs):
 310         assert substring is None or isinstance(substring, str)
 311         assert response_substring is None or isinstance(response_substring, str)
 312         d = defer.maybeDeferred(callable, *args, **kwargs)
 313         def done(res):
 314             if isinstance(res, failure.Failure):
 315                 res.trap(expected_failure)
 316                 if substring:
 317                     self.failUnless(substring in str(res),
 318                                     "%s: substring '%s' not in '%s'"
 319                                     % (which, substring, str(res)))
 320                 if response_substring:
 321                     self.failUnless(response_substring in res.value.response,
 322                                     "%s: response substring '%s' not in '%s'"
 323                                     % (which,
 324                                        response_substring, res.value.response))
 325             else:
 326                 self.fail("%s was supposed to raise %s, not get '%s'" %
 327                           (which, expected_failure, res))
 328         d.addBoth(done)
 329         return d
 330
 331     def should404(self, res, which):
 332         if isinstance(res, failure.Failure):
 333             res.trap(error.Error)
 334             self.failUnlessEqual(res.value.status, "404")
 335         else:
 336             self.fail("%s was supposed to Error(404), not get '%s'" %
 337                       (which, res))
 338
 339     def shouldHTTPError(self, res, which, code=None, substring=None,
 340                         response_substring=None):
 341         if isinstance(res, failure.Failure):
 342             res.trap(error.Error)
 343             if code is not None:
 344                 self.failUnlessEqual(res.value.status, str(code))
 345             if substring:
 346                 self.failUnless(substring in str(res),
 347                                 "substring '%s' not in '%s'"
 348                                 % (substring, str(res)))
 349             if response_substring:
 350                 self.failUnless(response_substring in res.value.response,
 351                                 "response substring '%s' not in '%s'"
 352                                 % (response_substring, res.value.response))
 353         else:
 354             self.fail("%s was supposed to Error(%s), not get '%s'" %
 355                       (which, code, res))
 356
 357     def shouldHTTPError2(self, which,
 358                          code=None, substring=None, response_substring=None,
 359                          callable=None, *args, **kwargs):
 360         assert substring is None or isinstance(substring, str)
 361         assert callable
 362         d = defer.maybeDeferred(callable, *args, **kwargs)
 363         d.addBoth(self.shouldHTTPError, which,
 364                   code, substring, response_substring)
 365         return d
 366
 367
 368 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 369     def test_create(self):
 370         pass
 371
 372     def test_welcome(self):
 373         d = self.GET("/")
 374         def _check(res):
 375             self.failUnless('Welcome To AllMyData' in res)
 376             self.failUnless('Tahoe' in res)
 377
 378             self.s.basedir = 'web/test_welcome'
 379             fileutil.make_dirs("web/test_welcome")
 380             fileutil.make_dirs("web/test_welcome/private")
 381             return self.GET("/")
 382         d.addCallback(_check)
 383         return d
 384
 385     def test_provisioning_math(self):
 386         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 387         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 388         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 389         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 390         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 391
 392     def test_provisioning(self):
 393         d = self.GET("/provisioning/")
 394         def _check(res):
 395             self.failUnless('Tahoe Provisioning Tool' in res)
 396             fields = {'filled': True,
 397                       "num_users": int(50e3),
 398                       "files_per_user": 1000,
 399                       "space_per_user": int(1e9),
 400                       "sharing_ratio": 1.0,
 401                       "encoding_parameters": "3-of-10-5",
 402                       "num_servers": 30,
 403                       "ownership_mode": "A",
 404                       "download_rate": 100,
 405                       "upload_rate": 10,
 406                       "delete_rate": 10,
 407                       "lease_timer": 7,
 408                       }
 409             return self.POST("/provisioning/", **fields)
 410
 411         d.addCallback(_check)
 412         def _check2(res):
 413             self.failUnless('Tahoe Provisioning Tool' in res)
 414             self.failUnless("Share space consumed: 167.01TB" in res)
 415
 416             fields = {'filled': True,
 417                       "num_users": int(50e6),
 418                       "files_per_user": 1000,
 419                       "space_per_user": int(5e9),
 420                       "sharing_ratio": 1.0,
 421                       "encoding_parameters": "25-of-100-50",
 422                       "num_servers": 30000,
 423                       "ownership_mode": "E",
 424                       "drive_failure_model": "U",
 425                       "drive_size": 1000,
 426                       "download_rate": 1000,
 427                       "upload_rate": 100,
 428                       "delete_rate": 100,
 429                       "lease_timer": 7,
 430                       }
 431             return self.POST("/provisioning/", **fields)
 432         d.addCallback(_check2)
 433         def _check3(res):
 434             self.failUnless("Share space consumed: huge!" in res)
 435             fields = {'filled': True}
 436             return self.POST("/provisioning/", **fields)
 437         d.addCallback(_check3)
 438         def _check4(res):
 439             self.failUnless("Share space consumed:" in res)
 440         d.addCallback(_check4)
 441         return d
 442
 443     def test_status(self):
 444         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 445         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 446         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 447         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 448         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 449         d = self.GET("/status", followRedirect=True)
 450         def _check(res):
 451             self.failUnless('Upload and Download Status' in res, res)
 452             self.failUnless('"down-%d"' % dl_num in res, res)
 453             self.failUnless('"up-%d"' % ul_num in res, res)
 454             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 455             self.failUnless('"publish-%d"' % pub_num in res, res)
 456             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 457         d.addCallback(_check)
 458         d.addCallback(lambda res: self.GET("/status/?t=json"))
 459         def _check_json(res):
 460             data = simplejson.loads(res)
 461             self.failUnless(isinstance(data, dict))
 462             active = data["active"]
 463             # TODO: test more. We need a way to fake an active operation
 464             # here.
 465         d.addCallback(_check_json)
 466
 467         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 468         def _check_dl(res):
 469             self.failUnless("File Download Status" in res, res)
 470         d.addCallback(_check_dl)
 471         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 472         def _check_ul(res):
 473             self.failUnless("File Upload Status" in res, res)
 474         d.addCallback(_check_ul)
 475         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 476         def _check_mapupdate(res):
 477             self.failUnless("Mutable File Servermap Update Status" in res, res)
 478         d.addCallback(_check_mapupdate)
 479         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 480         def _check_publish(res):
 481             self.failUnless("Mutable File Publish Status" in res, res)
 482         d.addCallback(_check_publish)
 483         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 484         def _check_retrieve(res):
 485             self.failUnless("Mutable File Retrieve Status" in res, res)
 486         d.addCallback(_check_retrieve)
 487
 488         return d
 489
 490     def test_status_numbers(self):
 491         drrm = status.DownloadResultsRendererMixin()
 492         self.failUnlessEqual(drrm.render_time(None, None), "")
 493         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 494         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 495         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 496         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 497         self.failUnlessEqual(drrm.render_rate(None, None), "")
 498         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 499         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 500         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 501
 502         urrm = status.UploadResultsRendererMixin()
 503         self.failUnlessEqual(urrm.render_time(None, None), "")
 504         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 505         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 506         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 507         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 508         self.failUnlessEqual(urrm.render_rate(None, None), "")
 509         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 510         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 511         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 512
 513     def test_GET_FILEURL(self):
 514         d = self.GET(self.public_url + "/foo/bar.txt")
 515         d.addCallback(self.failUnlessIsBarDotTxt)
 516         return d
 517
 518     def test_HEAD_FILEURL(self):
 519         d = self.HEAD(self.public_url + "/foo/bar.txt")
 520         def _got(headers):
 521             self.failUnlessEqual(headers["content-length"][0],
 522                                  str(len(self.BAR_CONTENTS)))
 523             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 524         d.addCallback(_got)
 525         return d
 526
 527     def test_GET_FILEURL_named(self):
 528         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 529         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 530         d = self.GET(base + "/@@name=/blah.txt")
 531         d.addCallback(self.failUnlessIsBarDotTxt)
 532         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 533         d.addCallback(self.failUnlessIsBarDotTxt)
 534         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 535         d.addCallback(self.failUnlessIsBarDotTxt)
 536         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 537         d.addCallback(self.failUnlessIsBarDotTxt)
 538         save_url = base + "?save=true&filename=blah.txt"
 539         d.addCallback(lambda res: self.GET(save_url))
 540         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 541         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 542         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 543         u_url = base + "?save=true&filename=" + u_fn_e
 544         d.addCallback(lambda res: self.GET(u_url))
 545         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 546         return d
 547
 548     def test_PUT_FILEURL_named_bad(self):
 549         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 550         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 551                              "400 Bad Request",
 552                              "/file can only be used with GET or HEAD",
 553                              self.PUT, base + "/@@name=/blah.txt", "")
 554         return d
 555
 556     def test_GET_DIRURL_named_bad(self):
 557         base = "/file/%s" % urllib.quote(self._foo_uri)
 558         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 559                              "400 Bad Request",
 560                              "is not a file-cap",
 561                              self.GET, base + "/@@name=/blah.txt")
 562         return d
 563
 564     def test_GET_slash_file_bad(self):
 565         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 566                              "404 Not Found",
 567                              "/file must be followed by a file-cap and a name",
 568                              self.GET, "/file")
 569         return d
 570
 571     def test_GET_unhandled_URI_named(self):
 572         contents, n, newuri = self.makefile(12)
 573         verifier_cap = n.get_verifier().to_string()
 574         base = "/file/%s" % urllib.quote(verifier_cap)
 575         # client.create_node_from_uri() can't handle verify-caps
 576         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 577                              "400 Bad Request",
 578                              "is not a valid file- or directory- cap",
 579                              self.GET, base)
 580         return d
 581
 582     def test_GET_unhandled_URI(self):
 583         contents, n, newuri = self.makefile(12)
 584         verifier_cap = n.get_verifier().to_string()
 585         base = "/uri/%s" % urllib.quote(verifier_cap)
 586         # client.create_node_from_uri() can't handle verify-caps
 587         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 588                              "400 Bad Request",
 589                              "is not a valid file- or directory- cap",
 590                              self.GET, base)
 591         return d
 592
 593     def test_GET_FILE_URI(self):
 594         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 595         d = self.GET(base)
 596         d.addCallback(self.failUnlessIsBarDotTxt)
 597         return d
 598
 599     def test_GET_FILE_URI_badchild(self):
 600         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 601         errmsg = "Files have no children, certainly not named 'boguschild'"
 602         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 603                              "400 Bad Request", errmsg,
 604                              self.GET, base)
 605         return d
 606
 607     def test_PUT_FILE_URI_badchild(self):
 608         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 609         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 610         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 611                              "400 Bad Request", errmsg,
 612                              self.PUT, base, "")
 613         return d
 614
 615     def test_GET_FILEURL_save(self):
 616         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 617         # TODO: look at the headers, expect a Content-Disposition: attachment
 618         # header.
 619         d.addCallback(self.failUnlessIsBarDotTxt)
 620         return d
 621
 622     def test_GET_FILEURL_missing(self):
 623         d = self.GET(self.public_url + "/foo/missing")
 624         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 625         return d
 626
 627     def test_PUT_NEWFILEURL(self):
 628         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 629         # TODO: we lose the response code, so we can't check this
 630         #self.failUnlessEqual(responsecode, 201)
 631         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 632         d.addCallback(lambda res:
 633                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 634                                                       self.NEWFILE_CONTENTS))
 635         return d
 636
 637     def test_PUT_NEWFILEURL_mutable(self):
 638         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 639                      self.NEWFILE_CONTENTS)
 640         # TODO: we lose the response code, so we can't check this
 641         #self.failUnlessEqual(responsecode, 201)
 642         def _check_uri(res):
 643             u = uri.from_string_mutable_filenode(res)
 644             self.failUnless(u.is_mutable())
 645             self.failIf(u.is_readonly())
 646             return res
 647         d.addCallback(_check_uri)
 648         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 649         d.addCallback(lambda res:
 650                       self.failUnlessMutableChildContentsAre(self._foo_node,
 651                                                              u"new.txt",
 652                                                              self.NEWFILE_CONTENTS))
 653         return d
 654
 655     def test_PUT_NEWFILEURL_mutable_toobig(self):
 656         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 657                              "413 Request Entity Too Large",
 658                              "SDMF is limited to one segment, and 10001 > 10000",
 659                              self.PUT,
 660                              self.public_url + "/foo/new.txt?mutable=true",
 661                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 662         return d
 663
 664     def test_PUT_NEWFILEURL_replace(self):
 665         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 666         # TODO: we lose the response code, so we can't check this
 667         #self.failUnlessEqual(responsecode, 200)
 668         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 669         d.addCallback(lambda res:
 670                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 671                                                       self.NEWFILE_CONTENTS))
 672         return d
 673
 674     def test_PUT_NEWFILEURL_bad_t(self):
 675         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 676                              "PUT to a file: bad t=bogus",
 677                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 678                              "contents")
 679         return d
 680
 681     def test_PUT_NEWFILEURL_no_replace(self):
 682         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 683                      self.NEWFILE_CONTENTS)
 684         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 685                   "409 Conflict",
 686                   "There was already a child by that name, and you asked me "
 687                   "to not replace it")
 688         return d
 689
 690     def test_PUT_NEWFILEURL_mkdirs(self):
 691         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 692         fn = self._foo_node
 693         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 694         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 695         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 696         d.addCallback(lambda res:
 697                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 698                                                       self.NEWFILE_CONTENTS))
 699         return d
 700
 701     def test_PUT_NEWFILEURL_blocked(self):
 702         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 703                      self.NEWFILE_CONTENTS)
 704         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 705                   "409 Conflict",
 706                   "Unable to create directory 'blockingfile': a file was in the way")
 707         return d
 708
 709     def test_DELETE_FILEURL(self):
 710         d = self.DELETE(self.public_url + "/foo/bar.txt")
 711         d.addCallback(lambda res:
 712                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 713         return d
 714
 715     def test_DELETE_FILEURL_missing(self):
 716         d = self.DELETE(self.public_url + "/foo/missing")
 717         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 718         return d
 719
 720     def test_DELETE_FILEURL_missing2(self):
 721         d = self.DELETE(self.public_url + "/missing/missing")
 722         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 723         return d
 724
 725     def test_GET_FILEURL_json(self):
 726         # twisted.web.http.parse_qs ignores any query args without an '=', so
 727         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 728         # instead. This may make it tricky to emulate the S3 interface
 729         # completely.
 730         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 731         d.addCallback(self.failUnlessIsBarJSON)
 732         return d
 733
 734     def test_GET_FILEURL_json_missing(self):
 735         d = self.GET(self.public_url + "/foo/missing?json")
 736         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 737         return d
 738
 739     def test_GET_FILEURL_uri(self):
 740         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 741         def _check(res):
 742             self.failUnlessEqual(res, self._bar_txt_uri)
 743         d.addCallback(_check)
 744         d.addCallback(lambda res:
 745                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 746         def _check2(res):
 747             # for now, for files, uris and readonly-uris are the same
 748             self.failUnlessEqual(res, self._bar_txt_uri)
 749         d.addCallback(_check2)
 750         return d
 751
 752     def test_GET_FILEURL_badtype(self):
 753         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 754                                   "bad t=bogus",
 755                                   self.GET,
 756                                   self.public_url + "/foo/bar.txt?t=bogus")
 757         return d
 758
 759     def test_GET_FILEURL_uri_missing(self):
 760         d = self.GET(self.public_url + "/foo/missing?t=uri")
 761         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 762         return d
 763
 764     def test_GET_DIRURL(self):
 765         # the addSlash means we get a redirect here
 766         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 767         ROOT = "../../.."
 768         d = self.GET(self.public_url + "/foo", followRedirect=True)
 769         def _check(res):
 770             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 771                             in res, res)
 772             # the FILE reference points to a URI, but it should end in bar.txt
 773             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 774                        (ROOT, urllib.quote(self._bar_txt_uri)))
 775             get_bar = "".join([r'<td>',
 776                                r'<a href="%s">bar.txt</a>' % bar_url,
 777                                r'</td>',
 778                                r'\s+<td>FILE</td>',
 779                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 780                                ])
 781             self.failUnless(re.search(get_bar, res), res)
 782             for line in res.split("\n"):
 783                 # find the line that contains the delete button for bar.txt
 784                 if ("form action" in line and
 785                     'value="delete"' in line and
 786                     'value="bar.txt"' in line):
 787                     # the form target should use a relative URL
 788                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 789                     self.failUnless(('action="%s"' % foo_url) in line, line)
 790                     # and the when_done= should too
 791                     #done_url = urllib.quote(???)
 792                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 793                     #                in line, line)
 794                     break
 795             else:
 796                 self.fail("unable to find delete-bar.txt line", res)
 797
 798             # the DIR reference just points to a URI
 799             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 800             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 801                        + r'\s+<td>DIR</td>')
 802             self.failUnless(re.search(get_sub, res), res)
 803         d.addCallback(_check)
 804
 805         # look at a directory which is readonly
 806         d.addCallback(lambda res:
 807                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 808         def _check2(res):
 809             self.failUnless("(readonly)" in res, res)
 810             self.failIf("Upload a file" in res, res)
 811         d.addCallback(_check2)
 812
 813         # and at a directory that contains a readonly directory
 814         d.addCallback(lambda res:
 815                       self.GET(self.public_url, followRedirect=True))
 816         def _check3(res):
 817             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 818                                       '</td>\s+<td>DIR-RO</td>', res))
 819         d.addCallback(_check3)
 820
 821         return d
 822
 823     def test_GET_DIRURL_badtype(self):
 824         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 825                                   400, "Bad Request",
 826                                   "bad t=bogus",
 827                                   self.GET,
 828                                   self.public_url + "/foo?t=bogus")
 829         return d
 830
 831     def test_GET_DIRURL_json(self):
 832         d = self.GET(self.public_url + "/foo?t=json")
 833         d.addCallback(self.failUnlessIsFooJSON)
 834         return d
 835
 836
 837     def test_POST_DIRURL_manifest_no_ophandle(self):
 838         d = self.shouldFail2(error.Error,
 839                              "test_POST_DIRURL_manifest_no_ophandle",
 840                              "400 Bad Request",
 841                              "slow operation requires ophandle=",
 842                              self.POST, self.public_url, t="start-manifest")
 843         return d
 844
 845     def test_POST_DIRURL_manifest(self):
 846         d = defer.succeed(None)
 847         def getman(ignored, output):
 848             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 849                           followRedirect=True)
 850             d.addCallback(self.wait_for_operation, "125")
 851             d.addCallback(self.get_operation_results, "125", output)
 852             return d
 853         d.addCallback(getman, None)
 854         def _got_html(manifest):
 855             self.failUnless("Manifest of SI=" in manifest)
 856             self.failUnless("<td>sub</td>" in manifest)
 857             self.failUnless(self._sub_uri in manifest)
 858             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 859         d.addCallback(_got_html)
 860
 861         # both t=status and unadorned GET should be identical
 862         d.addCallback(lambda res: self.GET("/operations/125"))
 863         d.addCallback(_got_html)
 864
 865         d.addCallback(getman, "html")
 866         d.addCallback(_got_html)
 867         d.addCallback(getman, "text")
 868         def _got_text(manifest):
 869             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 870             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 871         d.addCallback(_got_text)
 872         d.addCallback(getman, "JSON")
 873         def _got_json(manifest):
 874             data = manifest["manifest"]
 875             got = {}
 876             for (path_list, cap) in data:
 877                 got[tuple(path_list)] = cap
 878             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 879             self.failUnless((u"sub",u"baz.txt") in got)
 880         d.addCallback(_got_json)
 881         return d
 882
 883     def test_POST_DIRURL_deepsize_no_ophandle(self):
 884         d = self.shouldFail2(error.Error,
 885                              "test_POST_DIRURL_deepsize_no_ophandle",
 886                              "400 Bad Request",
 887                              "slow operation requires ophandle=",
 888                              self.POST, self.public_url, t="start-deep-size")
 889         return d
 890
 891     def test_POST_DIRURL_deepsize(self):
 892         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 893                       followRedirect=True)
 894         d.addCallback(self.wait_for_operation, "126")
 895         d.addCallback(self.get_operation_results, "126", "json")
 896         def _got_json(data):
 897             self.failUnlessEqual(data["finished"], True)
 898             size = data["size"]
 899             self.failUnless(size > 1000)
 900         d.addCallback(_got_json)
 901         d.addCallback(self.get_operation_results, "126", "text")
 902         def _got_text(res):
 903             mo = re.search(r'^size: (\d+)$', res, re.M)
 904             self.failUnless(mo, res)
 905             size = int(mo.group(1))
 906             # with directories, the size varies.
 907             self.failUnless(size > 1000)
 908         d.addCallback(_got_text)
 909         return d
 910
 911     def test_POST_DIRURL_deepstats_no_ophandle(self):
 912         d = self.shouldFail2(error.Error,
 913                              "test_POST_DIRURL_deepstats_no_ophandle",
 914                              "400 Bad Request",
 915                              "slow operation requires ophandle=",
 916                              self.POST, self.public_url, t="start-deep-stats")
 917         return d
 918
 919     def test_POST_DIRURL_deepstats(self):
 920         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
 921                       followRedirect=True)
 922         d.addCallback(self.wait_for_operation, "127")
 923         d.addCallback(self.get_operation_results, "127", "json")
 924         def _got_json(stats):
 925             expected = {"count-immutable-files": 3,
 926                         "count-mutable-files": 0,
 927                         "count-literal-files": 0,
 928                         "count-files": 3,
 929                         "count-directories": 3,
 930                         "size-immutable-files": 57,
 931                         "size-literal-files": 0,
 932                         #"size-directories": 1912, # varies
 933                         #"largest-directory": 1590,
 934                         "largest-directory-children": 5,
 935                         "largest-immutable-file": 19,
 936                         }
 937             for k,v in expected.iteritems():
 938                 self.failUnlessEqual(stats[k], v,
 939                                      "stats[%s] was %s, not %s" %
 940                                      (k, stats[k], v))
 941             self.failUnlessEqual(stats["size-files-histogram"],
 942                                  [ [11, 31, 3] ])
 943         d.addCallback(_got_json)
 944         return d
 945
 946     def test_GET_DIRURL_uri(self):
 947         d = self.GET(self.public_url + "/foo?t=uri")
 948         def _check(res):
 949             self.failUnlessEqual(res, self._foo_uri)
 950         d.addCallback(_check)
 951         return d
 952
 953     def test_GET_DIRURL_readonly_uri(self):
 954         d = self.GET(self.public_url + "/foo?t=readonly-uri")
 955         def _check(res):
 956             self.failUnlessEqual(res, self._foo_readonly_uri)
 957         d.addCallback(_check)
 958         return d
 959
 960     def test_PUT_NEWDIRURL(self):
 961         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
 962         d.addCallback(lambda res:
 963                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
 964         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
 965         d.addCallback(self.failUnlessNodeKeysAre, [])
 966         return d
 967
 968     def test_PUT_NEWDIRURL_exists(self):
 969         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
 970         d.addCallback(lambda res:
 971                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 972         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 973         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 974         return d
 975
 976     def test_PUT_NEWDIRURL_blocked(self):
 977         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
 978                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
 979                              self.PUT,
 980                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
 981         d.addCallback(lambda res:
 982                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 983         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 984         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 985         return d
 986
 987     def test_PUT_NEWDIRURL_mkdir_p(self):
 988         d = defer.succeed(None)
 989         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
 990         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
 991         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
 992         def mkdir_p(mkpnode):
 993             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
 994             d = self.POST(url)
 995             def made_subsub(ssuri):
 996                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
 997                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
 998                 d = self.POST(url)
 999                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1000                 return d
1001             d.addCallback(made_subsub)
1002             return d
1003         d.addCallback(mkdir_p)
1004         return d
1005
1006     def test_PUT_NEWDIRURL_mkdirs(self):
1007         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1008         d.addCallback(lambda res:
1009                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1010         d.addCallback(lambda res:
1011                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1012         d.addCallback(lambda res:
1013                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1014         d.addCallback(self.failUnlessNodeKeysAre, [])
1015         return d
1016
1017     def test_DELETE_DIRURL(self):
1018         d = self.DELETE(self.public_url + "/foo")
1019         d.addCallback(lambda res:
1020                       self.failIfNodeHasChild(self.public_root, u"foo"))
1021         return d
1022
1023     def test_DELETE_DIRURL_missing(self):
1024         d = self.DELETE(self.public_url + "/foo/missing")
1025         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1026         d.addCallback(lambda res:
1027                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1028         return d
1029
1030     def test_DELETE_DIRURL_missing2(self):
1031         d = self.DELETE(self.public_url + "/missing")
1032         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1033         return d
1034
1035     def dump_root(self):
1036         print "NODEWALK"
1037         w = webish.DirnodeWalkerMixin()
1038         def visitor(childpath, childnode, metadata):
1039             print childpath
1040         d = w.walk(self.public_root, visitor)
1041         return d
1042
1043     def failUnlessNodeKeysAre(self, node, expected_keys):
1044         for k in expected_keys:
1045             assert isinstance(k, unicode)
1046         d = node.list()
1047         def _check(children):
1048             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1049         d.addCallback(_check)
1050         return d
1051     def failUnlessNodeHasChild(self, node, name):
1052         assert isinstance(name, unicode)
1053         d = node.list()
1054         def _check(children):
1055             self.failUnless(name in children)
1056         d.addCallback(_check)
1057         return d
1058     def failIfNodeHasChild(self, node, name):
1059         assert isinstance(name, unicode)
1060         d = node.list()
1061         def _check(children):
1062             self.failIf(name in children)
1063         d.addCallback(_check)
1064         return d
1065
1066     def failUnlessChildContentsAre(self, node, name, expected_contents):
1067         assert isinstance(name, unicode)
1068         d = node.get_child_at_path(name)
1069         d.addCallback(lambda node: node.download_to_data())
1070         def _check(contents):
1071             self.failUnlessEqual(contents, expected_contents)
1072         d.addCallback(_check)
1073         return d
1074
1075     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1076         assert isinstance(name, unicode)
1077         d = node.get_child_at_path(name)
1078         d.addCallback(lambda node: node.download_best_version())
1079         def _check(contents):
1080             self.failUnlessEqual(contents, expected_contents)
1081         d.addCallback(_check)
1082         return d
1083
1084     def failUnlessChildURIIs(self, node, name, expected_uri):
1085         assert isinstance(name, unicode)
1086         d = node.get_child_at_path(name)
1087         def _check(child):
1088             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1089         d.addCallback(_check)
1090         return d
1091
1092     def failUnlessURIMatchesChild(self, got_uri, node, name):
1093         assert isinstance(name, unicode)
1094         d = node.get_child_at_path(name)
1095         def _check(child):
1096             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1097         d.addCallback(_check)
1098         return d
1099
1100     def failUnlessCHKURIHasContents(self, got_uri, contents):
1101         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1102
1103     def test_POST_upload(self):
1104         d = self.POST(self.public_url + "/foo", t="upload",
1105                       file=("new.txt", self.NEWFILE_CONTENTS))
1106         fn = self._foo_node
1107         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1108         d.addCallback(lambda res:
1109                       self.failUnlessChildContentsAre(fn, u"new.txt",
1110                                                       self.NEWFILE_CONTENTS))
1111         return d
1112
1113     def test_POST_upload_unicode(self):
1114         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1115         d = self.POST(self.public_url + "/foo", t="upload",
1116                       file=(filename, self.NEWFILE_CONTENTS))
1117         fn = self._foo_node
1118         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1119         d.addCallback(lambda res:
1120                       self.failUnlessChildContentsAre(fn, filename,
1121                                                       self.NEWFILE_CONTENTS))
1122         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1123         d.addCallback(lambda res: self.GET(target_url))
1124         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1125                                                             self.NEWFILE_CONTENTS,
1126                                                             contents))
1127         return d
1128
1129     def test_POST_upload_unicode_named(self):
1130         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1131         d = self.POST(self.public_url + "/foo", t="upload",
1132                       name=filename,
1133                       file=("overridden", self.NEWFILE_CONTENTS))
1134         fn = self._foo_node
1135         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1136         d.addCallback(lambda res:
1137                       self.failUnlessChildContentsAre(fn, filename,
1138                                                       self.NEWFILE_CONTENTS))
1139         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1140         d.addCallback(lambda res: self.GET(target_url))
1141         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1142                                                             self.NEWFILE_CONTENTS,
1143                                                             contents))
1144         return d
1145
1146     def test_POST_upload_no_link(self):
1147         d = self.POST("/uri", t="upload",
1148                       file=("new.txt", self.NEWFILE_CONTENTS))
1149         def _check_upload_results(page):
1150             # this should be a page which describes the results of the upload
1151             # that just finished.
1152             self.failUnless("Upload Results:" in page)
1153             self.failUnless("URI:" in page)
1154             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1155             mo = uri_re.search(page)
1156             self.failUnless(mo, page)
1157             new_uri = mo.group(1)
1158             return new_uri
1159         d.addCallback(_check_upload_results)
1160         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1161         return d
1162
1163     def test_POST_upload_no_link_whendone(self):
1164         d = self.POST("/uri", t="upload", when_done="/",
1165                       file=("new.txt", self.NEWFILE_CONTENTS))
1166         d.addBoth(self.shouldRedirect, "/")
1167         return d
1168
1169     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1170         d = defer.maybeDeferred(callable, *args, **kwargs)
1171         def done(res):
1172             if isinstance(res, failure.Failure):
1173                 res.trap(error.PageRedirect)
1174                 statuscode = res.value.status
1175                 target = res.value.location
1176                 return checker(statuscode, target)
1177             self.fail("%s: callable was supposed to redirect, not return '%s'"
1178                       % (which, res))
1179         d.addBoth(done)
1180         return d
1181
1182     def test_POST_upload_no_link_whendone_results(self):
1183         def check(statuscode, target):
1184             self.failUnlessEqual(statuscode, str(http.FOUND))
1185             self.failUnless(target.startswith(self.webish_url), target)
1186             return client.getPage(target, method="GET")
1187         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1188                                  check,
1189                                  self.POST, "/uri", t="upload",
1190                                  when_done="/uri/%(uri)s",
1191                                  file=("new.txt", self.NEWFILE_CONTENTS))
1192         d.addCallback(lambda res:
1193                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1194         return d
1195
1196     def test_POST_upload_no_link_mutable(self):
1197         d = self.POST("/uri", t="upload", mutable="true",
1198                       file=("new.txt", self.NEWFILE_CONTENTS))
1199         def _check(new_uri):
1200             new_uri = new_uri.strip()
1201             self.new_uri = new_uri
1202             u = IURI(new_uri)
1203             self.failUnless(IMutableFileURI.providedBy(u))
1204             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1205             n = self.s.create_node_from_uri(new_uri)
1206             return n.download_best_version()
1207         d.addCallback(_check)
1208         def _check2(data):
1209             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1210             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1211         d.addCallback(_check2)
1212         def _check3(data):
1213             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1214             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1215         d.addCallback(_check3)
1216         def _check4(data):
1217             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1218         d.addCallback(_check4)
1219         return d
1220
1221     def test_POST_upload_no_link_mutable_toobig(self):
1222         d = self.shouldFail2(error.Error,
1223                              "test_POST_upload_no_link_mutable_toobig",
1224                              "413 Request Entity Too Large",
1225                              "SDMF is limited to one segment, and 10001 > 10000",
1226                              self.POST,
1227                              "/uri", t="upload", mutable="true",
1228                              file=("new.txt",
1229                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1230         return d
1231
1232     def test_POST_upload_mutable(self):
1233         # this creates a mutable file
1234         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1235                       file=("new.txt", self.NEWFILE_CONTENTS))
1236         fn = self._foo_node
1237         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1238         d.addCallback(lambda res:
1239                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1240                                                              self.NEWFILE_CONTENTS))
1241         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1242         def _got(newnode):
1243             self.failUnless(IMutableFileNode.providedBy(newnode))
1244             self.failUnless(newnode.is_mutable())
1245             self.failIf(newnode.is_readonly())
1246             self._mutable_node = newnode
1247             self._mutable_uri = newnode.get_uri()
1248         d.addCallback(_got)
1249
1250         # now upload it again and make sure that the URI doesn't change
1251         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1252         d.addCallback(lambda res:
1253                       self.POST(self.public_url + "/foo", t="upload",
1254                                 mutable="true",
1255                                 file=("new.txt", NEWER_CONTENTS)))
1256         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1257         d.addCallback(lambda res:
1258                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1259                                                              NEWER_CONTENTS))
1260         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1261         def _got2(newnode):
1262             self.failUnless(IMutableFileNode.providedBy(newnode))
1263             self.failUnless(newnode.is_mutable())
1264             self.failIf(newnode.is_readonly())
1265             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1266         d.addCallback(_got2)
1267
1268         # upload a second time, using PUT instead of POST
1269         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1270         d.addCallback(lambda res:
1271                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1272         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1273         d.addCallback(lambda res:
1274                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1275                                                              NEW2_CONTENTS))
1276
1277         # finally list the directory, since mutable files are displayed
1278         # slightly differently
1279
1280         d.addCallback(lambda res:
1281                       self.GET(self.public_url + "/foo/",
1282                                followRedirect=True))
1283         def _check_page(res):
1284             # TODO: assert more about the contents
1285             self.failUnless("SSK" in res)
1286             return res
1287         d.addCallback(_check_page)
1288
1289         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1290         def _got3(newnode):
1291             self.failUnless(IMutableFileNode.providedBy(newnode))
1292             self.failUnless(newnode.is_mutable())
1293             self.failIf(newnode.is_readonly())
1294             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1295         d.addCallback(_got3)
1296
1297         # look at the JSON form of the enclosing directory
1298         d.addCallback(lambda res:
1299                       self.GET(self.public_url + "/foo/?t=json",
1300                                followRedirect=True))
1301         def _check_page_json(res):
1302             parsed = simplejson.loads(res)
1303             self.failUnlessEqual(parsed[0], "dirnode")
1304             children = dict( [(unicode(name),value)
1305                               for (name,value)
1306                               in parsed[1]["children"].iteritems()] )
1307             self.failUnless("new.txt" in children)
1308             new_json = children["new.txt"]
1309             self.failUnlessEqual(new_json[0], "filenode")
1310             self.failUnless(new_json[1]["mutable"])
1311             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1312             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1313             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1314         d.addCallback(_check_page_json)
1315
1316         # and the JSON form of the file
1317         d.addCallback(lambda res:
1318                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1319         def _check_file_json(res):
1320             parsed = simplejson.loads(res)
1321             self.failUnlessEqual(parsed[0], "filenode")
1322             self.failUnless(parsed[1]["mutable"])
1323             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1324             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1325             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1326         d.addCallback(_check_file_json)
1327
1328         # and look at t=uri and t=readonly-uri
1329         d.addCallback(lambda res:
1330                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1331         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1332         d.addCallback(lambda res:
1333                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1334         def _check_ro_uri(res):
1335             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1336             self.failUnlessEqual(res, ro_uri)
1337         d.addCallback(_check_ro_uri)
1338
1339         # make sure we can get to it from /uri/URI
1340         d.addCallback(lambda res:
1341                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1342         d.addCallback(lambda res:
1343                       self.failUnlessEqual(res, NEW2_CONTENTS))
1344
1345         # and that HEAD computes the size correctly
1346         d.addCallback(lambda res:
1347                       self.HEAD(self.public_url + "/foo/new.txt"))
1348         def _got_headers(headers):
1349             self.failUnlessEqual(headers["content-length"][0],
1350                                  str(len(NEW2_CONTENTS)))
1351             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1352         d.addCallback(_got_headers)
1353
1354         # make sure that size errors are displayed correctly for overwrite
1355         d.addCallback(lambda res:
1356                       self.shouldFail2(error.Error,
1357                                        "test_POST_upload_mutable-toobig",
1358                                        "413 Request Entity Too Large",
1359                                        "SDMF is limited to one segment, and 10001 > 10000",
1360                                        self.POST,
1361                                        self.public_url + "/foo", t="upload",
1362                                        mutable="true",
1363                                        file=("new.txt",
1364                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1365                                        ))
1366
1367         d.addErrback(self.dump_error)
1368         return d
1369
1370     def test_POST_upload_mutable_toobig(self):
1371         d = self.shouldFail2(error.Error,
1372                              "test_POST_upload_no_link_mutable_toobig",
1373                              "413 Request Entity Too Large",
1374                              "SDMF is limited to one segment, and 10001 > 10000",
1375                              self.POST,
1376                              self.public_url + "/foo",
1377                              t="upload", mutable="true",
1378                              file=("new.txt",
1379                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1380         return d
1381
1382     def dump_error(self, f):
1383         # if the web server returns an error code (like 400 Bad Request),
1384         # web.client.getPage puts the HTTP response body into the .response
1385         # attribute of the exception object that it gives back. It does not
1386         # appear in the Failure's repr(), so the ERROR that trial displays
1387         # will be rather terse and unhelpful. addErrback this method to the
1388         # end of your chain to get more information out of these errors.
1389         if f.check(error.Error):
1390             print "web.error.Error:"
1391             print f
1392             print f.value.response
1393         return f
1394
1395     def test_POST_upload_replace(self):
1396         d = self.POST(self.public_url + "/foo", t="upload",
1397                       file=("bar.txt", self.NEWFILE_CONTENTS))
1398         fn = self._foo_node
1399         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1400         d.addCallback(lambda res:
1401                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1402                                                       self.NEWFILE_CONTENTS))
1403         return d
1404
1405     def test_POST_upload_no_replace_ok(self):
1406         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1407                       file=("new.txt", self.NEWFILE_CONTENTS))
1408         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1409         d.addCallback(lambda res: self.failUnlessEqual(res,
1410                                                        self.NEWFILE_CONTENTS))
1411         return d
1412
1413     def test_POST_upload_no_replace_queryarg(self):
1414         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1415                       file=("bar.txt", self.NEWFILE_CONTENTS))
1416         d.addBoth(self.shouldFail, error.Error,
1417                   "POST_upload_no_replace_queryarg",
1418                   "409 Conflict",
1419                   "There was already a child by that name, and you asked me "
1420                   "to not replace it")
1421         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1422         d.addCallback(self.failUnlessIsBarDotTxt)
1423         return d
1424
1425     def test_POST_upload_no_replace_field(self):
1426         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1427                       file=("bar.txt", self.NEWFILE_CONTENTS))
1428         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1429                   "409 Conflict",
1430                   "There was already a child by that name, and you asked me "
1431                   "to not replace it")
1432         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1433         d.addCallback(self.failUnlessIsBarDotTxt)
1434         return d
1435
1436     def test_POST_upload_whendone(self):
1437         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1438                       file=("new.txt", self.NEWFILE_CONTENTS))
1439         d.addBoth(self.shouldRedirect, "/THERE")
1440         fn = self._foo_node
1441         d.addCallback(lambda res:
1442                       self.failUnlessChildContentsAre(fn, u"new.txt",
1443                                                       self.NEWFILE_CONTENTS))
1444         return d
1445
1446     def test_POST_upload_named(self):
1447         fn = self._foo_node
1448         d = self.POST(self.public_url + "/foo", t="upload",
1449                       name="new.txt", file=self.NEWFILE_CONTENTS)
1450         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1451         d.addCallback(lambda res:
1452                       self.failUnlessChildContentsAre(fn, u"new.txt",
1453                                                       self.NEWFILE_CONTENTS))
1454         return d
1455
1456     def test_POST_upload_named_badfilename(self):
1457         d = self.POST(self.public_url + "/foo", t="upload",
1458                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1459         d.addBoth(self.shouldFail, error.Error,
1460                   "test_POST_upload_named_badfilename",
1461                   "400 Bad Request",
1462                   "name= may not contain a slash",
1463                   )
1464         # make sure that nothing was added
1465         d.addCallback(lambda res:
1466                       self.failUnlessNodeKeysAre(self._foo_node,
1467                                                  [u"bar.txt", u"blockingfile",
1468                                                   u"empty", u"n\u00fc.txt",
1469                                                   u"sub"]))
1470         return d
1471
1472     def test_POST_FILEURL_check(self):
1473         bar_url = self.public_url + "/foo/bar.txt"
1474         d = self.POST(bar_url, t="check")
1475         def _check(res):
1476             self.failUnless("Healthy!" in res)
1477         d.addCallback(_check)
1478         redir_url = "http://allmydata.org/TARGET"
1479         def _check2(statuscode, target):
1480             self.failUnlessEqual(statuscode, str(http.FOUND))
1481             self.failUnlessEqual(target, redir_url)
1482         d.addCallback(lambda res:
1483                       self.shouldRedirect2("test_POST_FILEURL_check",
1484                                            _check2,
1485                                            self.POST, bar_url,
1486                                            t="check",
1487                                            when_done=redir_url))
1488         d.addCallback(lambda res:
1489                       self.POST(bar_url, t="check", return_to=redir_url))
1490         def _check3(res):
1491             self.failUnless("Healthy!" in res)
1492             self.failUnless("Return to parent directory" in res)
1493             self.failUnless(redir_url in res)
1494         d.addCallback(_check3)
1495
1496         d.addCallback(lambda res:
1497                       self.POST(bar_url, t="check", output="JSON"))
1498         def _check_json(res):
1499             data = simplejson.loads(res)
1500             self.failUnless("storage-index" in data)
1501             self.failUnless(data["results"]["healthy"])
1502         d.addCallback(_check_json)
1503
1504         return d
1505
1506     def test_POST_FILEURL_check_and_repair(self):
1507         bar_url = self.public_url + "/foo/bar.txt"
1508         d = self.POST(bar_url, t="check", repair="true")
1509         def _check(res):
1510             self.failUnless("Healthy!" in res)
1511         d.addCallback(_check)
1512         redir_url = "http://allmydata.org/TARGET"
1513         def _check2(statuscode, target):
1514             self.failUnlessEqual(statuscode, str(http.FOUND))
1515             self.failUnlessEqual(target, redir_url)
1516         d.addCallback(lambda res:
1517                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1518                                            _check2,
1519                                            self.POST, bar_url,
1520                                            t="check", repair="true",
1521                                            when_done=redir_url))
1522         d.addCallback(lambda res:
1523                       self.POST(bar_url, t="check", return_to=redir_url))
1524         def _check3(res):
1525             self.failUnless("Healthy!" in res)
1526             self.failUnless("Return to parent directory" in res)
1527             self.failUnless(redir_url in res)
1528         d.addCallback(_check3)
1529         return d
1530
1531     def test_POST_DIRURL_check(self):
1532         foo_url = self.public_url + "/foo/"
1533         d = self.POST(foo_url, t="check")
1534         def _check(res):
1535             self.failUnless("Healthy!" in res)
1536         d.addCallback(_check)
1537         redir_url = "http://allmydata.org/TARGET"
1538         def _check2(statuscode, target):
1539             self.failUnlessEqual(statuscode, str(http.FOUND))
1540             self.failUnlessEqual(target, redir_url)
1541         d.addCallback(lambda res:
1542                       self.shouldRedirect2("test_POST_DIRURL_check",
1543                                            _check2,
1544                                            self.POST, foo_url,
1545                                            t="check",
1546                                            when_done=redir_url))
1547         d.addCallback(lambda res:
1548                       self.POST(foo_url, t="check", return_to=redir_url))
1549         def _check3(res):
1550             self.failUnless("Healthy!" in res)
1551             self.failUnless("Return to parent directory" in res)
1552             self.failUnless(redir_url in res)
1553         d.addCallback(_check3)
1554
1555         d.addCallback(lambda res:
1556                       self.POST(foo_url, t="check", output="JSON"))
1557         def _check_json(res):
1558             data = simplejson.loads(res)
1559             self.failUnless("storage-index" in data)
1560             self.failUnless(data["results"]["healthy"])
1561         d.addCallback(_check_json)
1562
1563         return d
1564
1565     def test_POST_DIRURL_check_and_repair(self):
1566         foo_url = self.public_url + "/foo/"
1567         d = self.POST(foo_url, t="check", repair="true")
1568         def _check(res):
1569             self.failUnless("Healthy!" in res)
1570         d.addCallback(_check)
1571         redir_url = "http://allmydata.org/TARGET"
1572         def _check2(statuscode, target):
1573             self.failUnlessEqual(statuscode, str(http.FOUND))
1574             self.failUnlessEqual(target, redir_url)
1575         d.addCallback(lambda res:
1576                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1577                                            _check2,
1578                                            self.POST, foo_url,
1579                                            t="check", repair="true",
1580                                            when_done=redir_url))
1581         d.addCallback(lambda res:
1582                       self.POST(foo_url, t="check", return_to=redir_url))
1583         def _check3(res):
1584             self.failUnless("Healthy!" in res)
1585             self.failUnless("Return to parent directory" in res)
1586             self.failUnless(redir_url in res)
1587         d.addCallback(_check3)
1588         return d
1589
1590     def wait_for_operation(self, ignored, ophandle):
1591         url = "/operations/" + ophandle
1592         url += "?t=status&output=JSON"
1593         d = self.GET(url)
1594         def _got(res):
1595             data = simplejson.loads(res)
1596             if not data["finished"]:
1597                 d = self.stall(delay=1.0)
1598                 d.addCallback(self.wait_for_operation, ophandle)
1599                 return d
1600             return data
1601         d.addCallback(_got)
1602         return d
1603
1604     def get_operation_results(self, ignored, ophandle, output=None):
1605         url = "/operations/" + ophandle
1606         url += "?t=status"
1607         if output:
1608             url += "&output=" + output
1609         d = self.GET(url)
1610         def _got(res):
1611             if output and output.lower() == "json":
1612                 return simplejson.loads(res)
1613             return res
1614         d.addCallback(_got)
1615         return d
1616
1617     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1618         d = self.shouldFail2(error.Error,
1619                              "test_POST_DIRURL_deepcheck_no_ophandle",
1620                              "400 Bad Request",
1621                              "slow operation requires ophandle=",
1622                              self.POST, self.public_url, t="start-deep-check")
1623         return d
1624
1625     def test_POST_DIRURL_deepcheck(self):
1626         def _check_redirect(statuscode, target):
1627             self.failUnlessEqual(statuscode, str(http.FOUND))
1628             self.failUnless(target.endswith("/operations/123"))
1629         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1630                                  self.POST, self.public_url,
1631                                  t="start-deep-check", ophandle="123")
1632         d.addCallback(self.wait_for_operation, "123")
1633         def _check_json(data):
1634             self.failUnlessEqual(data["finished"], True)
1635             self.failUnlessEqual(data["count-objects-checked"], 8)
1636             self.failUnlessEqual(data["count-objects-healthy"], 8)
1637         d.addCallback(_check_json)
1638         d.addCallback(self.get_operation_results, "123", "html")
1639         def _check_html(res):
1640             self.failUnless("Objects Checked: <span>8</span>" in res)
1641             self.failUnless("Objects Healthy: <span>8</span>" in res)
1642         d.addCallback(_check_html)
1643         d.addCallback(lambda res:
1644                       self.shouldFail2(error.Error, "one", "404 Not Found",
1645                                        "No detailed results for SI bogus",
1646                                        self.GET, "/operations/123/bogus"))
1647         foo_si = self._foo_node.get_storage_index()
1648         foo_si_s = base32.b2a(foo_si)
1649         d.addCallback(lambda res:
1650                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1651         def _check_foo_json(res):
1652             data = simplejson.loads(res)
1653             self.failUnlessEqual(data["storage-index"], foo_si_s)
1654             self.failUnless(data["results"]["healthy"])
1655         d.addCallback(_check_foo_json)
1656         return d
1657
1658     def test_POST_DIRURL_deepcheck_and_repair(self):
1659         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1660                       ophandle="124", output="json", followRedirect=True)
1661         d.addCallback(self.wait_for_operation, "124")
1662         def _check_json(data):
1663             self.failUnlessEqual(data["finished"], True)
1664             self.failUnlessEqual(data["count-objects-checked"], 8)
1665             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1666             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1667             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1668             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1669             self.failUnlessEqual(data["count-repairs-successful"], 0)
1670             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1671             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1672             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1673             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1674         d.addCallback(_check_json)
1675         d.addCallback(self.get_operation_results, "124", "html")
1676         def _check_html(res):
1677             self.failUnless("Objects Checked: <span>8</span>" in res)
1678
1679             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1680             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1681             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1682
1683             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1684             self.failUnless("Repairs Successful: <span>0</span>" in res)
1685             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1686
1687             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1688             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1689             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1690         d.addCallback(_check_html)
1691         return d
1692
1693     def test_POST_FILEURL_bad_t(self):
1694         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1695                              "POST to file: bad t=bogus",
1696                              self.POST, self.public_url + "/foo/bar.txt",
1697                              t="bogus")
1698         return d
1699
1700     def test_POST_mkdir(self): # return value?
1701         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1702         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1703         d.addCallback(self.failUnlessNodeKeysAre, [])
1704         return d
1705
1706     def test_POST_mkdir_2(self):
1707         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1708         d.addCallback(lambda res:
1709                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1710         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1711         d.addCallback(self.failUnlessNodeKeysAre, [])
1712         return d
1713
1714     def test_POST_mkdirs_2(self):
1715         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1716         d.addCallback(lambda res:
1717                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1718         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1719         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1720         d.addCallback(self.failUnlessNodeKeysAre, [])
1721         return d
1722
1723     def test_POST_mkdir_no_parentdir_noredirect(self):
1724         d = self.POST("/uri?t=mkdir")
1725         def _after_mkdir(res):
1726             uri.NewDirectoryURI.init_from_string(res)
1727         d.addCallback(_after_mkdir)
1728         return d
1729
1730     def test_POST_mkdir_no_parentdir_redirect(self):
1731         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1732         d.addBoth(self.shouldRedirect, None, statuscode='303')
1733         def _check_target(target):
1734             target = urllib.unquote(target)
1735             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1736         d.addCallback(_check_target)
1737         return d
1738
1739     def test_POST_noparent_bad(self):
1740         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1741                                   "/uri accepts only PUT, PUT?t=mkdir, "
1742                                   "POST?t=upload, and POST?t=mkdir",
1743                                   self.POST, "/uri?t=bogus")
1744         return d
1745
1746     def test_welcome_page_mkdir_button(self):
1747         # Fetch the welcome page.
1748         d = self.GET("/")
1749         def _after_get_welcome_page(res):
1750             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1751             mo = MKDIR_BUTTON_RE.search(res)
1752             formaction = mo.group(1)
1753             formt = mo.group(2)
1754             formaname = mo.group(3)
1755             formavalue = mo.group(4)
1756             return (formaction, formt, formaname, formavalue)
1757         d.addCallback(_after_get_welcome_page)
1758         def _after_parse_form(res):
1759             (formaction, formt, formaname, formavalue) = res
1760             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1761         d.addCallback(_after_parse_form)
1762         d.addBoth(self.shouldRedirect, None, statuscode='303')
1763         return d
1764
1765     def test_POST_mkdir_replace(self): # return value?
1766         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1767         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1768         d.addCallback(self.failUnlessNodeKeysAre, [])
1769         return d
1770
1771     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1772         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1773         d.addBoth(self.shouldFail, error.Error,
1774                   "POST_mkdir_no_replace_queryarg",
1775                   "409 Conflict",
1776                   "There was already a child by that name, and you asked me "
1777                   "to not replace it")
1778         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1779         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1780         return d
1781
1782     def test_POST_mkdir_no_replace_field(self): # return value?
1783         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1784                       replace="false")
1785         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1786                   "409 Conflict",
1787                   "There was already a child by that name, and you asked me "
1788                   "to not replace it")
1789         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1790         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1791         return d
1792
1793     def test_POST_mkdir_whendone_field(self):
1794         d = self.POST(self.public_url + "/foo",
1795                       t="mkdir", name="newdir", when_done="/THERE")
1796         d.addBoth(self.shouldRedirect, "/THERE")
1797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798         d.addCallback(self.failUnlessNodeKeysAre, [])
1799         return d
1800
1801     def test_POST_mkdir_whendone_queryarg(self):
1802         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1803                       t="mkdir", name="newdir")
1804         d.addBoth(self.shouldRedirect, "/THERE")
1805         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1806         d.addCallback(self.failUnlessNodeKeysAre, [])
1807         return d
1808
1809     def test_POST_bad_t(self):
1810         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1811                              "POST to a directory with bad t=BOGUS",
1812                              self.POST, self.public_url + "/foo", t="BOGUS")
1813         return d
1814
1815     def test_POST_set_children(self):
1816         contents9, n9, newuri9 = self.makefile(9)
1817         contents10, n10, newuri10 = self.makefile(10)
1818         contents11, n11, newuri11 = self.makefile(11)
1819
1820         reqbody = """{
1821                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1822                                                 "size": 0,
1823                                                 "metadata": {
1824                                                   "ctime": 1002777696.7564139,
1825                                                   "mtime": 1002777696.7564139
1826                                                  }
1827                                                } ],
1828                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1829                                                 "size": 1,
1830                                                 "metadata": {
1831                                                   "ctime": 1002777696.7564139,
1832                                                   "mtime": 1002777696.7564139
1833                                                  }
1834                                                } ],
1835                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1836                                                 "size": 2,
1837                                                 "metadata": {
1838                                                   "ctime": 1002777696.7564139,
1839                                                   "mtime": 1002777696.7564139
1840                                                  }
1841                                                } ]
1842                     }""" % (newuri9, newuri10, newuri11)
1843
1844         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1845
1846         d = client.getPage(url, method="POST", postdata=reqbody)
1847         def _then(res):
1848             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1849             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1850             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1851
1852         d.addCallback(_then)
1853         d.addErrback(self.dump_error)
1854         return d
1855
1856     def test_POST_put_uri(self):
1857         contents, n, newuri = self.makefile(8)
1858         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1859         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1860         d.addCallback(lambda res:
1861                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1862                                                       contents))
1863         return d
1864
1865     def test_POST_put_uri_replace(self):
1866         contents, n, newuri = self.makefile(8)
1867         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1868         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1869         d.addCallback(lambda res:
1870                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1871                                                       contents))
1872         return d
1873
1874     def test_POST_put_uri_no_replace_queryarg(self):
1875         contents, n, newuri = self.makefile(8)
1876         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1877                       name="bar.txt", uri=newuri)
1878         d.addBoth(self.shouldFail, error.Error,
1879                   "POST_put_uri_no_replace_queryarg",
1880                   "409 Conflict",
1881                   "There was already a child by that name, and you asked me "
1882                   "to not replace it")
1883         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1884         d.addCallback(self.failUnlessIsBarDotTxt)
1885         return d
1886
1887     def test_POST_put_uri_no_replace_field(self):
1888         contents, n, newuri = self.makefile(8)
1889         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1890                       name="bar.txt", uri=newuri)
1891         d.addBoth(self.shouldFail, error.Error,
1892                   "POST_put_uri_no_replace_field",
1893                   "409 Conflict",
1894                   "There was already a child by that name, and you asked me "
1895                   "to not replace it")
1896         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1897         d.addCallback(self.failUnlessIsBarDotTxt)
1898         return d
1899
1900     def test_POST_delete(self):
1901         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1902         d.addCallback(lambda res: self._foo_node.list())
1903         def _check(children):
1904             self.failIf(u"bar.txt" in children)
1905         d.addCallback(_check)
1906         return d
1907
1908     def test_POST_rename_file(self):
1909         d = self.POST(self.public_url + "/foo", t="rename",
1910                       from_name="bar.txt", to_name='wibble.txt')
1911         d.addCallback(lambda res:
1912                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1913         d.addCallback(lambda res:
1914                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1915         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1916         d.addCallback(self.failUnlessIsBarDotTxt)
1917         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1918         d.addCallback(self.failUnlessIsBarJSON)
1919         return d
1920
1921     def test_POST_rename_file_redundant(self):
1922         d = self.POST(self.public_url + "/foo", t="rename",
1923                       from_name="bar.txt", to_name='bar.txt')
1924         d.addCallback(lambda res:
1925                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1926         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1927         d.addCallback(self.failUnlessIsBarDotTxt)
1928         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
1929         d.addCallback(self.failUnlessIsBarJSON)
1930         return d
1931
1932     def test_POST_rename_file_replace(self):
1933         # rename a file and replace a directory with it
1934         d = self.POST(self.public_url + "/foo", t="rename",
1935                       from_name="bar.txt", to_name='empty')
1936         d.addCallback(lambda res:
1937                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1938         d.addCallback(lambda res:
1939                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
1940         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
1941         d.addCallback(self.failUnlessIsBarDotTxt)
1942         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1943         d.addCallback(self.failUnlessIsBarJSON)
1944         return d
1945
1946     def test_POST_rename_file_no_replace_queryarg(self):
1947         # rename a file and replace a directory with it
1948         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
1949                       from_name="bar.txt", to_name='empty')
1950         d.addBoth(self.shouldFail, error.Error,
1951                   "POST_rename_file_no_replace_queryarg",
1952                   "409 Conflict",
1953                   "There was already a child by that name, and you asked me "
1954                   "to not replace it")
1955         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1956         d.addCallback(self.failUnlessIsEmptyJSON)
1957         return d
1958
1959     def test_POST_rename_file_no_replace_field(self):
1960         # rename a file and replace a directory with it
1961         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
1962                       from_name="bar.txt", to_name='empty')
1963         d.addBoth(self.shouldFail, error.Error,
1964                   "POST_rename_file_no_replace_field",
1965                   "409 Conflict",
1966                   "There was already a child by that name, and you asked me "
1967                   "to not replace it")
1968         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1969         d.addCallback(self.failUnlessIsEmptyJSON)
1970         return d
1971
1972     def failUnlessIsEmptyJSON(self, res):
1973         data = simplejson.loads(res)
1974         self.failUnlessEqual(data[0], "dirnode", data)
1975         self.failUnlessEqual(len(data[1]["children"]), 0)
1976
1977     def test_POST_rename_file_slash_fail(self):
1978         d = self.POST(self.public_url + "/foo", t="rename",
1979                       from_name="bar.txt", to_name='kirk/spock.txt')
1980         d.addBoth(self.shouldFail, error.Error,
1981                   "test_POST_rename_file_slash_fail",
1982                   "400 Bad Request",
1983                   "to_name= may not contain a slash",
1984                   )
1985         d.addCallback(lambda res:
1986                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1987         return d
1988
1989     def test_POST_rename_dir(self):
1990         d = self.POST(self.public_url, t="rename",
1991                       from_name="foo", to_name='plunk')
1992         d.addCallback(lambda res:
1993                       self.failIfNodeHasChild(self.public_root, u"foo"))
1994         d.addCallback(lambda res:
1995                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
1996         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
1997         d.addCallback(self.failUnlessIsFooJSON)
1998         return d
1999
2000     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2001         """ If target is not None then the redirection has to go to target.  If
2002         statuscode is not None then the redirection has to be accomplished with
2003         that HTTP status code."""
2004         if not isinstance(res, failure.Failure):
2005             to_where = (target is None) and "somewhere" or ("to " + target)
2006             self.fail("%s: we were expecting to get redirected %s, not get an"
2007                       " actual page: %s" % (which, to_where, res))
2008         res.trap(error.PageRedirect)
2009         if statuscode is not None:
2010             self.failUnlessEqual(res.value.status, statuscode,
2011                                  "%s: not a redirect" % which)
2012         if target is not None:
2013             # the PageRedirect does not seem to capture the uri= query arg
2014             # properly, so we can't check for it.
2015             realtarget = self.webish_url + target
2016             self.failUnlessEqual(res.value.location, realtarget,
2017                                  "%s: wrong target" % which)
2018         return res.value.location
2019
2020     def test_GET_URI_form(self):
2021         base = "/uri?uri=%s" % self._bar_txt_uri
2022         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2023         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2024         d = self.GET(base)
2025         d.addBoth(self.shouldRedirect, targetbase)
2026         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2027         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2028         d.addCallback(lambda res: self.GET(base+"&t=json"))
2029         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2030         d.addCallback(self.log, "about to get file by uri")
2031         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2032         d.addCallback(self.failUnlessIsBarDotTxt)
2033         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2034         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2035                                            followRedirect=True))
2036         d.addCallback(self.failUnlessIsFooJSON)
2037         d.addCallback(self.log, "got dir by uri")
2038
2039         return d
2040
2041     def test_GET_URI_form_bad(self):
2042         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2043                              "400 Bad Request", "GET /uri requires uri=",
2044                              self.GET, "/uri")
2045         return d
2046
2047     def test_GET_rename_form(self):
2048         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2049                      followRedirect=True)
2050         def _check(res):
2051             self.failUnless('name="when_done" value="."' in res, res)
2052             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2053         d.addCallback(_check)
2054         return d
2055
2056     def log(self, res, msg):
2057         #print "MSG: %s  RES: %s" % (msg, res)
2058         log.msg(msg)
2059         return res
2060
2061     def test_GET_URI_URL(self):
2062         base = "/uri/%s" % self._bar_txt_uri
2063         d = self.GET(base)
2064         d.addCallback(self.failUnlessIsBarDotTxt)
2065         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2066         d.addCallback(self.failUnlessIsBarDotTxt)
2067         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2068         d.addCallback(self.failUnlessIsBarDotTxt)
2069         return d
2070
2071     def test_GET_URI_URL_dir(self):
2072         base = "/uri/%s?t=json" % self._foo_uri
2073         d = self.GET(base)
2074         d.addCallback(self.failUnlessIsFooJSON)
2075         return d
2076
2077     def test_GET_URI_URL_missing(self):
2078         base = "/uri/%s" % self._bad_file_uri
2079         d = self.GET(base)
2080         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2081                   http.GONE, response_substring="NotEnoughSharesError")
2082         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2083         # here? we must arrange for a download to fail after target.open()
2084         # has been called, and then inspect the response to see that it is
2085         # shorter than we expected.
2086         return d
2087
2088     def test_PUT_NEWFILEURL_uri(self):
2089         contents, n, new_uri = self.makefile(8)
2090         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2091         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2092         d.addCallback(lambda res:
2093                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2094                                                       contents))
2095         return d
2096
2097     def test_PUT_NEWFILEURL_uri_replace(self):
2098         contents, n, new_uri = self.makefile(8)
2099         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2100         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2101         d.addCallback(lambda res:
2102                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2103                                                       contents))
2104         return d
2105
2106     def test_PUT_NEWFILEURL_uri_no_replace(self):
2107         contents, n, new_uri = self.makefile(8)
2108         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2109         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2110                   "409 Conflict",
2111                   "There was already a child by that name, and you asked me "
2112                   "to not replace it")
2113         return d
2114
2115     def test_PUT_NEWFILE_URI(self):
2116         file_contents = "New file contents here\n"
2117         d = self.PUT("/uri", file_contents)
2118         def _check(uri):
2119             self.failUnless(uri in FakeCHKFileNode.all_contents)
2120             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2121                                  file_contents)
2122             return self.GET("/uri/%s" % uri)
2123         d.addCallback(_check)
2124         def _check2(res):
2125             self.failUnlessEqual(res, file_contents)
2126         d.addCallback(_check2)
2127         return d
2128
2129     def test_PUT_NEWFILE_URI_only_PUT(self):
2130         d = self.PUT("/uri?t=bogus", "")
2131         d.addBoth(self.shouldFail, error.Error,
2132                   "PUT_NEWFILE_URI_only_PUT",
2133                   "400 Bad Request",
2134                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2135         return d
2136
2137     def test_PUT_NEWFILE_URI_mutable(self):
2138         file_contents = "New file contents here\n"
2139         d = self.PUT("/uri?mutable=true", file_contents)
2140         def _check_mutable(uri):
2141             uri = uri.strip()
2142             u = IURI(uri)
2143             self.failUnless(IMutableFileURI.providedBy(u))
2144             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2145             n = self.s.create_node_from_uri(uri)
2146             return n.download_best_version()
2147         d.addCallback(_check_mutable)
2148         def _check2_mutable(data):
2149             self.failUnlessEqual(data, file_contents)
2150         d.addCallback(_check2_mutable)
2151         return d
2152
2153         def _check(uri):
2154             self.failUnless(uri in FakeCHKFileNode.all_contents)
2155             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2156                                  file_contents)
2157             return self.GET("/uri/%s" % uri)
2158         d.addCallback(_check)
2159         def _check2(res):
2160             self.failUnlessEqual(res, file_contents)
2161         d.addCallback(_check2)
2162         return d
2163
2164     def test_PUT_mkdir(self):
2165         d = self.PUT("/uri?t=mkdir", "")
2166         def _check(uri):
2167             n = self.s.create_node_from_uri(uri.strip())
2168             d2 = self.failUnlessNodeKeysAre(n, [])
2169             d2.addCallback(lambda res:
2170                            self.GET("/uri/%s?t=json" % uri))
2171             return d2
2172         d.addCallback(_check)
2173         d.addCallback(self.failUnlessIsEmptyJSON)
2174         return d
2175
2176     def test_POST_check(self):
2177         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2178         def _done(res):
2179             # this returns a string form of the results, which are probably
2180             # None since we're using fake filenodes.
2181             # TODO: verify that the check actually happened, by changing
2182             # FakeCHKFileNode to count how many times .check() has been
2183             # called.
2184             pass
2185         d.addCallback(_done)
2186         return d
2187
2188     def test_bad_method(self):
2189         url = self.webish_url + self.public_url + "/foo/bar.txt"
2190         d = self.shouldHTTPError2("test_bad_method",
2191                                   501, "Not Implemented",
2192                                   "I don't know how to treat a BOGUS request.",
2193                                   client.getPage, url, method="BOGUS")
2194         return d
2195
2196     def test_short_url(self):
2197         url = self.webish_url + "/uri"
2198         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2199                                   "I don't know how to treat a DELETE request.",
2200                                   client.getPage, url, method="DELETE")
2201         return d
2202
2203     def test_ophandle_bad(self):
2204         url = self.webish_url + "/operations/bogus?t=status"
2205         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2206                                   "unknown/expired handle 'bogus'",
2207                                   client.getPage, url)
2208         return d
2209
2210     def test_ophandle_cancel(self):
2211         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2212                       followRedirect=True)
2213         d.addCallback(lambda ignored:
2214                       self.GET("/operations/128?t=status&output=JSON"))
2215         def _check1(res):
2216             data = simplejson.loads(res)
2217             self.failUnless("finished" in data, res)
2218             monitor = self.ws.root.child_operations.handles["128"][0]
2219             d = self.POST("/operations/128?t=cancel&output=JSON")
2220             def _check2(res):
2221                 data = simplejson.loads(res)
2222                 self.failUnless("finished" in data, res)
2223                 # t=cancel causes the handle to be forgotten
2224                 self.failUnless(monitor.is_cancelled())
2225             d.addCallback(_check2)
2226             return d
2227         d.addCallback(_check1)
2228         d.addCallback(lambda ignored:
2229                       self.shouldHTTPError2("test_ophandle_cancel",
2230                                             404, "404 Not Found",
2231                                             "unknown/expired handle '128'",
2232                                             self.GET,
2233                                             "/operations/128?t=status&output=JSON"))
2234         return d
2235
2236     def test_ophandle_retainfor(self):
2237         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2238                       followRedirect=True)
2239         d.addCallback(lambda ignored:
2240                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2241         def _check1(res):
2242             data = simplejson.loads(res)
2243             self.failUnless("finished" in data, res)
2244         d.addCallback(_check1)
2245         # the retain-for=0 will cause the handle to be expired very soon
2246         d.addCallback(self.stall, 2.0)
2247         d.addCallback(lambda ignored:
2248                       self.shouldHTTPError2("test_ophandle_retainfor",
2249                                             404, "404 Not Found",
2250                                             "unknown/expired handle '129'",
2251                                             self.GET,
2252                                             "/operations/129?t=status&output=JSON"))
2253         return d
2254
2255     def test_ophandle_release_after_complete(self):
2256         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2257                       followRedirect=True)
2258         d.addCallback(self.wait_for_operation, "130")
2259         d.addCallback(lambda ignored:
2260                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2261         # the release-after-complete=true will cause the handle to be expired
2262         d.addCallback(lambda ignored:
2263                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2264                                             404, "404 Not Found",
2265                                             "unknown/expired handle '130'",
2266                                             self.GET,
2267                                             "/operations/130?t=status&output=JSON"))
2268         return d
2269
2270     def test_incident(self):
2271         d = self.POST("/report_incident", details="eek")
2272         def _done(res):
2273             self.failUnless("Thank you for your report!" in res, res)
2274         d.addCallback(_done)
2275         return d
2276
2277
2278 class Util(unittest.TestCase):
2279     def test_abbreviate_time(self):
2280         self.failUnlessEqual(common.abbreviate_time(None), "")
2281         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2282         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2283         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2284         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2285
2286     def test_abbreviate_rate(self):
2287         self.failUnlessEqual(common.abbreviate_rate(None), "")
2288         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2289         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2290         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2291
2292     def test_abbreviate_size(self):
2293         self.failUnlessEqual(common.abbreviate_size(None), "")
2294         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2295         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2296         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2297         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2298