src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil, base32
  12 from allmydata.util.assertutil import precondition
  13 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  14      FakeMutableFileNode, create_chk_filenode
  15 from allmydata.interfaces import IURI, INewDirectoryURI, \
  16      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  17 from allmydata.mutable import servermap, publish, retrieve
  18 import common_util as testutil
  19
  20 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  21 # create a webserver that works against them
  22
  23 class FakeIntroducerClient:
  24     def get_all_connectors(self):
  25         return {}
  26     def get_all_connections_for(self, service_name):
  27         return frozenset()
  28     def get_all_peerids(self):
  29         return frozenset()
  30
  31 class FakeClient(service.MultiService):
  32     nodeid = "fake_nodeid"
  33     nickname = "fake_nickname"
  34     basedir = "fake_basedir"
  35     def get_versions(self):
  36         return {'allmydata': "fake",
  37                 'foolscap': "fake",
  38                 'twisted': "fake",
  39                 'zfec': "fake",
  40                 }
  41     introducer_furl = "None"
  42     introducer_client = FakeIntroducerClient()
  43     _all_upload_status = [upload.UploadStatus()]
  44     _all_download_status = [download.DownloadStatus()]
  45     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  46     _all_publish_statuses = [publish.PublishStatus()]
  47     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  48     convergence = "some random string"
  49
  50     def connected_to_introducer(self):
  51         return False
  52
  53     def get_nickname_for_peerid(self, peerid):
  54         return u"John Doe"
  55
  56     def get_permuted_peers(self, service_name, key):
  57         return []
  58
  59     def create_node_from_uri(self, auri):
  60         precondition(isinstance(auri, str), auri)
  61         u = uri.from_string(auri)
  62         if (INewDirectoryURI.providedBy(u)
  63             or IReadonlyNewDirectoryURI.providedBy(u)):
  64             return FakeDirectoryNode(self).init_from_uri(u)
  65         if IFileURI.providedBy(u):
  66             return FakeCHKFileNode(u, self)
  67         assert IMutableFileURI.providedBy(u), u
  68         return FakeMutableFileNode(self).init_from_uri(u)
  69
  70     def create_empty_dirnode(self):
  71         n = FakeDirectoryNode(self)
  72         d = n.create()
  73         d.addCallback(lambda res: n)
  74         return d
  75
  76     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  77     def create_mutable_file(self, contents=""):
  78         n = FakeMutableFileNode(self)
  79         return n.create(contents)
  80
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(self, data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92
  93     def list_all_upload_statuses(self):
  94         return self._all_upload_status
  95     def list_all_download_statuses(self):
  96         return self._all_download_status
  97     def list_all_mapupdate_statuses(self):
  98         return self._all_mapupdate_statuses
  99     def list_all_publish_statuses(self):
 100         return self._all_publish_statuses
 101     def list_all_retrieve_statuses(self):
 102         return self._all_retrieve_statuses
 103     def list_all_helper_statuses(self):
 104         return []
 105
 106 class MyGetter(client.HTTPPageGetter):
 107     handleStatus_206 = lambda self: self.handleStatus_200()
 108
 109 class HTTPClientHEADFactory(client.HTTPClientFactory):
 110     protocol = MyGetter
 111
 112     def noPage(self, reason):
 113         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 114         # exception when the response to a HEAD request had no body (when in
 115         # fact they are defined to never have a body). This was fixed in
 116         # Twisted-8.0 . To work around this, we catch the
 117         # PartialDownloadError and make it disappear.
 118         if (reason.check(client.PartialDownloadError)
 119             and self.method.upper() == "HEAD"):
 120             self.page("")
 121             return
 122         return client.HTTPClientFactory.noPage(self, reason)
 123
 124 class HTTPClientGETFactory(client.HTTPClientFactory):
 125     protocol = MyGetter
 126
 127 class WebMixin(object):
 128     def setUp(self):
 129         self.s = FakeClient()
 130         self.s.startService()
 131         self.staticdir = self.mktemp()
 132         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 133         s.setServiceParent(self.s)
 134         self.webish_port = port = s.listener._port.getHost().port
 135         self.webish_url = "http://localhost:%d" % port
 136
 137         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 138         d = defer.DeferredList(l)
 139         def _then(res):
 140             self.public_root = res[0][1]
 141             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 142             self.public_url = "/uri/" + self.public_root.get_uri()
 143             self.private_root = res[1][1]
 144
 145             foo = res[2][1]
 146             self._foo_node = foo
 147             self._foo_uri = foo.get_uri()
 148             self._foo_readonly_uri = foo.get_readonly_uri()
 149             self._foo_verifycap = foo.get_verify_cap().to_string()
 150             # NOTE: we ignore the deferred on all set_uri() calls, because we
 151             # know the fake nodes do these synchronously
 152             self.public_root.set_uri(u"foo", foo.get_uri())
 153
 154             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 155             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 156             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 157
 158             foo.set_uri(u"empty", res[3][1].get_uri())
 159             sub_uri = res[4][1].get_uri()
 160             self._sub_uri = sub_uri
 161             foo.set_uri(u"sub", sub_uri)
 162             sub = self.s.create_node_from_uri(sub_uri)
 163
 164             _ign, n, blocking_uri = self.makefile(1)
 165             foo.set_uri(u"blockingfile", blocking_uri)
 166
 167             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 168             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 169             # still think of it as an umlaut
 170             foo.set_uri(unicode_filename, self._bar_txt_uri)
 171
 172             _ign, n, baz_file = self.makefile(2)
 173             self._baz_file_uri = baz_file
 174             sub.set_uri(u"baz.txt", baz_file)
 175
 176             _ign, n, self._bad_file_uri = self.makefile(3)
 177             # this uri should not be downloadable
 178             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 179
 180             rodir = res[5][1]
 181             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 182             rodir.set_uri(u"nor", baz_file)
 183
 184             # public/
 185             # public/foo/
 186             # public/foo/bar.txt
 187             # public/foo/blockingfile
 188             # public/foo/empty/
 189             # public/foo/sub/
 190             # public/foo/sub/baz.txt
 191             # public/reedownlee/
 192             # public/reedownlee/nor
 193             self.NEWFILE_CONTENTS = "newfile contents\n"
 194
 195             return foo.get_metadata_for(u"bar.txt")
 196         d.addCallback(_then)
 197         def _got_metadata(metadata):
 198             self._bar_txt_metadata = metadata
 199         d.addCallback(_got_metadata)
 200         return d
 201
 202     def makefile(self, number):
 203         contents = "contents of file %s\n" % number
 204         n = create_chk_filenode(self.s, contents)
 205         return contents, n, n.get_uri()
 206
 207     def tearDown(self):
 208         return self.s.stopService()
 209
 210     def failUnlessIsBarDotTxt(self, res):
 211         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 212
 213     def failUnlessIsBarJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], u"filenode")
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failIf(data[1]["mutable"])
 219         self.failIf("rw_uri" in data[1]) # immutable
 220         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 221         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 222         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 223
 224     def failUnlessIsFooJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], "dirnode", res)
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failUnless(data[1]["mutable"])
 230         self.failUnless("rw_uri" in data[1]) # mutable
 231         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 232         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 233         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 234
 235         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 236         self.failUnlessEqual(kidnames,
 237                              [u"bar.txt", u"blockingfile", u"empty",
 238                               u"n\u00fc.txt", u"sub"])
 239         kids = dict( [(unicode(name),value)
 240                       for (name,value)
 241                       in data[1]["children"].iteritems()] )
 242         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 243         self.failUnless("metadata" in kids[u"sub"][1])
 244         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 245         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 246         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 247         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 248         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 249         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 250                              self._bar_txt_verifycap)
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 252                              self._bar_txt_metadata["ctime"])
 253         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 254                              self._bar_txt_uri)
 255
 256     def GET(self, urlpath, followRedirect=False, return_response=False,
 257             **kwargs):
 258         # if return_response=True, this fires with (data, statuscode,
 259         # respheaders) instead of just data.
 260         assert not isinstance(urlpath, unicode)
 261         url = self.webish_url + urlpath
 262         factory = HTTPClientGETFactory(url, method="GET",
 263                                        followRedirect=followRedirect, **kwargs)
 264         reactor.connectTCP("localhost", self.webish_port, factory)
 265         d = factory.deferred
 266         def _got_data(data):
 267             return (data, factory.status, factory.response_headers)
 268         if return_response:
 269             d.addCallback(_got_data)
 270         return factory.deferred
 271
 272     def HEAD(self, urlpath, return_response=False, **kwargs):
 273         # this requires some surgery, because twisted.web.client doesn't want
 274         # to give us back the response headers.
 275         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 276         reactor.connectTCP("localhost", self.webish_port, factory)
 277         d = factory.deferred
 278         def _got_data(data):
 279             return (data, factory.status, factory.response_headers)
 280         if return_response:
 281             d.addCallback(_got_data)
 282         return factory.deferred
 283
 284     def PUT(self, urlpath, data, **kwargs):
 285         url = self.webish_url + urlpath
 286         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 287
 288     def DELETE(self, urlpath):
 289         url = self.webish_url + urlpath
 290         return client.getPage(url, method="DELETE")
 291
 292     def POST(self, urlpath, followRedirect=False, **fields):
 293         url = self.webish_url + urlpath
 294         sepbase = "boogabooga"
 295         sep = "--" + sepbase
 296         form = []
 297         form.append(sep)
 298         form.append('Content-Disposition: form-data; name="_charset"')
 299         form.append('')
 300         form.append('UTF-8')
 301         form.append(sep)
 302         for name, value in fields.iteritems():
 303             if isinstance(value, tuple):
 304                 filename, value = value
 305                 form.append('Content-Disposition: form-data; name="%s"; '
 306                             'filename="%s"' % (name, filename.encode("utf-8")))
 307             else:
 308                 form.append('Content-Disposition: form-data; name="%s"' % name)
 309             form.append('')
 310             if isinstance(value, unicode):
 311                 value = value.encode("utf-8")
 312             else:
 313                 value = str(value)
 314             assert isinstance(value, str)
 315             form.append(value)
 316             form.append(sep)
 317         form[-1] += "--"
 318         body = "\r\n".join(form) + "\r\n"
 319         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 320                    }
 321         return client.getPage(url, method="POST", postdata=body,
 322                               headers=headers, followRedirect=followRedirect)
 323
 324     def shouldFail(self, res, expected_failure, which,
 325                    substring=None, response_substring=None):
 326         if isinstance(res, failure.Failure):
 327             res.trap(expected_failure)
 328             if substring:
 329                 self.failUnless(substring in str(res),
 330                                 "substring '%s' not in '%s'"
 331                                 % (substring, str(res)))
 332             if response_substring:
 333                 self.failUnless(response_substring in res.value.response,
 334                                 "response substring '%s' not in '%s'"
 335                                 % (response_substring, res.value.response))
 336         else:
 337             self.fail("%s was supposed to raise %s, not get '%s'" %
 338                       (which, expected_failure, res))
 339
 340     def shouldFail2(self, expected_failure, which, substring,
 341                     response_substring,
 342                     callable, *args, **kwargs):
 343         assert substring is None or isinstance(substring, str)
 344         assert response_substring is None or isinstance(response_substring, str)
 345         d = defer.maybeDeferred(callable, *args, **kwargs)
 346         def done(res):
 347             if isinstance(res, failure.Failure):
 348                 res.trap(expected_failure)
 349                 if substring:
 350                     self.failUnless(substring in str(res),
 351                                     "%s: substring '%s' not in '%s'"
 352                                     % (which, substring, str(res)))
 353                 if response_substring:
 354                     self.failUnless(response_substring in res.value.response,
 355                                     "%s: response substring '%s' not in '%s'"
 356                                     % (which,
 357                                        response_substring, res.value.response))
 358             else:
 359                 self.fail("%s was supposed to raise %s, not get '%s'" %
 360                           (which, expected_failure, res))
 361         d.addBoth(done)
 362         return d
 363
 364     def should404(self, res, which):
 365         if isinstance(res, failure.Failure):
 366             res.trap(error.Error)
 367             self.failUnlessEqual(res.value.status, "404")
 368         else:
 369             self.fail("%s was supposed to Error(404), not get '%s'" %
 370                       (which, res))
 371
 372     def shouldHTTPError(self, res, which, code=None, substring=None,
 373                         response_substring=None):
 374         if isinstance(res, failure.Failure):
 375             res.trap(error.Error)
 376             if code is not None:
 377                 self.failUnlessEqual(res.value.status, str(code))
 378             if substring:
 379                 self.failUnless(substring in str(res),
 380                                 "substring '%s' not in '%s'"
 381                                 % (substring, str(res)))
 382             if response_substring:
 383                 self.failUnless(response_substring in res.value.response,
 384                                 "response substring '%s' not in '%s'"
 385                                 % (response_substring, res.value.response))
 386         else:
 387             self.fail("%s was supposed to Error(%s), not get '%s'" %
 388                       (which, code, res))
 389
 390     def shouldHTTPError2(self, which,
 391                          code=None, substring=None, response_substring=None,
 392                          callable=None, *args, **kwargs):
 393         assert substring is None or isinstance(substring, str)
 394         assert callable
 395         d = defer.maybeDeferred(callable, *args, **kwargs)
 396         d.addBoth(self.shouldHTTPError, which,
 397                   code, substring, response_substring)
 398         return d
 399
 400
 401 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 402     def test_create(self):
 403         pass
 404
 405     def test_welcome(self):
 406         d = self.GET("/")
 407         def _check(res):
 408             self.failUnless('Welcome To AllMyData' in res)
 409             self.failUnless('Tahoe' in res)
 410
 411             self.s.basedir = 'web/test_welcome'
 412             fileutil.make_dirs("web/test_welcome")
 413             fileutil.make_dirs("web/test_welcome/private")
 414             return self.GET("/")
 415         d.addCallback(_check)
 416         return d
 417
 418     def test_provisioning(self):
 419         d = self.GET("/provisioning/")
 420         def _check(res):
 421             self.failUnless('Tahoe Provisioning Tool' in res)
 422             fields = {'filled': True,
 423                       "num_users": int(50e3),
 424                       "files_per_user": 1000,
 425                       "space_per_user": int(1e9),
 426                       "sharing_ratio": 1.0,
 427                       "encoding_parameters": "3-of-10-5",
 428                       "num_servers": 30,
 429                       "ownership_mode": "A",
 430                       "download_rate": 100,
 431                       "upload_rate": 10,
 432                       "delete_rate": 10,
 433                       "lease_timer": 7,
 434                       }
 435             return self.POST("/provisioning/", **fields)
 436
 437         d.addCallback(_check)
 438         def _check2(res):
 439             self.failUnless('Tahoe Provisioning Tool' in res)
 440             self.failUnless("Share space consumed: 167.01TB" in res)
 441
 442             fields = {'filled': True,
 443                       "num_users": int(50e6),
 444                       "files_per_user": 1000,
 445                       "space_per_user": int(5e9),
 446                       "sharing_ratio": 1.0,
 447                       "encoding_parameters": "25-of-100-50",
 448                       "num_servers": 30000,
 449                       "ownership_mode": "E",
 450                       "drive_failure_model": "U",
 451                       "drive_size": 1000,
 452                       "download_rate": 1000,
 453                       "upload_rate": 100,
 454                       "delete_rate": 100,
 455                       "lease_timer": 7,
 456                       }
 457             return self.POST("/provisioning/", **fields)
 458         d.addCallback(_check2)
 459         def _check3(res):
 460             self.failUnless("Share space consumed: huge!" in res)
 461             fields = {'filled': True}
 462             return self.POST("/provisioning/", **fields)
 463         d.addCallback(_check3)
 464         def _check4(res):
 465             self.failUnless("Share space consumed:" in res)
 466         d.addCallback(_check4)
 467         return d
 468
 469     def test_reliability_tool(self):
 470         try:
 471             from allmydata import reliability
 472             _hush_pyflakes = reliability
 473         except:
 474             raise unittest.SkipTest("reliability tool requires Numeric")
 475
 476         d = self.GET("/reliability/")
 477         def _check(res):
 478             self.failUnless('Tahoe Reliability Tool' in res)
 479             fields = {'drive_lifetime': "8Y",
 480                       "k": "3",
 481                       "R": "7",
 482                       "N": "10",
 483                       "delta": "100000",
 484                       "check_period": "1M",
 485                       "report_period": "3M",
 486                       "report_span": "5Y",
 487                       }
 488             return self.POST("/reliability/", **fields)
 489
 490         d.addCallback(_check)
 491         def _check2(res):
 492             self.failUnless('Tahoe Reliability Tool' in res)
 493             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 494             self.failUnless(re.search(r, res), res)
 495         d.addCallback(_check2)
 496         return d
 497
 498     def test_status(self):
 499         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 500         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 501         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 502         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 503         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 504         d = self.GET("/status", followRedirect=True)
 505         def _check(res):
 506             self.failUnless('Upload and Download Status' in res, res)
 507             self.failUnless('"down-%d"' % dl_num in res, res)
 508             self.failUnless('"up-%d"' % ul_num in res, res)
 509             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 510             self.failUnless('"publish-%d"' % pub_num in res, res)
 511             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 512         d.addCallback(_check)
 513         d.addCallback(lambda res: self.GET("/status/?t=json"))
 514         def _check_json(res):
 515             data = simplejson.loads(res)
 516             self.failUnless(isinstance(data, dict))
 517             active = data["active"]
 518             # TODO: test more. We need a way to fake an active operation
 519             # here.
 520         d.addCallback(_check_json)
 521
 522         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 523         def _check_dl(res):
 524             self.failUnless("File Download Status" in res, res)
 525         d.addCallback(_check_dl)
 526         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 527         def _check_ul(res):
 528             self.failUnless("File Upload Status" in res, res)
 529         d.addCallback(_check_ul)
 530         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 531         def _check_mapupdate(res):
 532             self.failUnless("Mutable File Servermap Update Status" in res, res)
 533         d.addCallback(_check_mapupdate)
 534         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 535         def _check_publish(res):
 536             self.failUnless("Mutable File Publish Status" in res, res)
 537         d.addCallback(_check_publish)
 538         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 539         def _check_retrieve(res):
 540             self.failUnless("Mutable File Retrieve Status" in res, res)
 541         d.addCallback(_check_retrieve)
 542
 543         return d
 544
 545     def test_status_numbers(self):
 546         drrm = status.DownloadResultsRendererMixin()
 547         self.failUnlessEqual(drrm.render_time(None, None), "")
 548         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 549         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 550         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 551         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 552         self.failUnlessEqual(drrm.render_rate(None, None), "")
 553         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 554         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 555         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 556
 557         urrm = status.UploadResultsRendererMixin()
 558         self.failUnlessEqual(urrm.render_time(None, None), "")
 559         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 560         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 561         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 562         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 563         self.failUnlessEqual(urrm.render_rate(None, None), "")
 564         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 565         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 566         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 567
 568     def test_GET_FILEURL(self):
 569         d = self.GET(self.public_url + "/foo/bar.txt")
 570         d.addCallback(self.failUnlessIsBarDotTxt)
 571         return d
 572
 573     def test_GET_FILEURL_range(self):
 574         headers = {"range": "bytes=1-10"}
 575         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 576                      return_response=True)
 577         def _got((res, status, headers)):
 578             self.failUnlessEqual(int(status), 206)
 579             self.failUnless(headers.has_key("content-range"))
 580             self.failUnlessEqual(headers["content-range"][0],
 581                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 582             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 583         d.addCallback(_got)
 584         return d
 585
 586     def test_GET_FILEURL_partial_range(self):
 587         headers = {"range": "bytes=5-"}
 588         length  = len(self.BAR_CONTENTS)
 589         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 590                      return_response=True)
 591         def _got((res, status, headers)):
 592             self.failUnlessEqual(int(status), 206)
 593             self.failUnless(headers.has_key("content-range"))
 594             self.failUnlessEqual(headers["content-range"][0],
 595                                  "bytes 5-%d/%d" % (length-1, length))
 596             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 597         d.addCallback(_got)
 598         return d
 599
 600     def test_HEAD_FILEURL_range(self):
 601         headers = {"range": "bytes=1-10"}
 602         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 603                      return_response=True)
 604         def _got((res, status, headers)):
 605             self.failUnlessEqual(res, "")
 606             self.failUnlessEqual(int(status), 206)
 607             self.failUnless(headers.has_key("content-range"))
 608             self.failUnlessEqual(headers["content-range"][0],
 609                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 610         d.addCallback(_got)
 611         return d
 612
 613     def test_HEAD_FILEURL_partial_range(self):
 614         headers = {"range": "bytes=5-"}
 615         length  = len(self.BAR_CONTENTS)
 616         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 617                      return_response=True)
 618         def _got((res, status, headers)):
 619             self.failUnlessEqual(int(status), 206)
 620             self.failUnless(headers.has_key("content-range"))
 621             self.failUnlessEqual(headers["content-range"][0],
 622                                  "bytes 5-%d/%d" % (length-1, length))
 623         d.addCallback(_got)
 624         return d
 625
 626     def test_GET_FILEURL_range_bad(self):
 627         headers = {"range": "BOGUS=fizbop-quarnak"}
 628         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 629                              "400 Bad Request",
 630                              "Syntactically invalid http range header",
 631                              self.GET, self.public_url + "/foo/bar.txt",
 632                              headers=headers)
 633         return d
 634
 635     def test_HEAD_FILEURL(self):
 636         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 637         def _got((res, status, headers)):
 638             self.failUnlessEqual(res, "")
 639             self.failUnlessEqual(headers["content-length"][0],
 640                                  str(len(self.BAR_CONTENTS)))
 641             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 642         d.addCallback(_got)
 643         return d
 644
 645     def test_GET_FILEURL_named(self):
 646         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 647         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 648         d = self.GET(base + "/@@name=/blah.txt")
 649         d.addCallback(self.failUnlessIsBarDotTxt)
 650         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 651         d.addCallback(self.failUnlessIsBarDotTxt)
 652         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 653         d.addCallback(self.failUnlessIsBarDotTxt)
 654         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 655         d.addCallback(self.failUnlessIsBarDotTxt)
 656         save_url = base + "?save=true&filename=blah.txt"
 657         d.addCallback(lambda res: self.GET(save_url))
 658         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 659         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 660         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 661         u_url = base + "?save=true&filename=" + u_fn_e
 662         d.addCallback(lambda res: self.GET(u_url))
 663         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 664         return d
 665
 666     def test_PUT_FILEURL_named_bad(self):
 667         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 668         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 669                              "400 Bad Request",
 670                              "/file can only be used with GET or HEAD",
 671                              self.PUT, base + "/@@name=/blah.txt", "")
 672         return d
 673
 674     def test_GET_DIRURL_named_bad(self):
 675         base = "/file/%s" % urllib.quote(self._foo_uri)
 676         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 677                              "400 Bad Request",
 678                              "is not a file-cap",
 679                              self.GET, base + "/@@name=/blah.txt")
 680         return d
 681
 682     def test_GET_slash_file_bad(self):
 683         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 684                              "404 Not Found",
 685                              "/file must be followed by a file-cap and a name",
 686                              self.GET, "/file")
 687         return d
 688
 689     def test_GET_unhandled_URI_named(self):
 690         contents, n, newuri = self.makefile(12)
 691         verifier_cap = n.get_verify_cap().to_string()
 692         base = "/file/%s" % urllib.quote(verifier_cap)
 693         # client.create_node_from_uri() can't handle verify-caps
 694         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 695                              "400 Bad Request",
 696                              "is not a valid file- or directory- cap",
 697                              self.GET, base)
 698         return d
 699
 700     def test_GET_unhandled_URI(self):
 701         contents, n, newuri = self.makefile(12)
 702         verifier_cap = n.get_verify_cap().to_string()
 703         base = "/uri/%s" % urllib.quote(verifier_cap)
 704         # client.create_node_from_uri() can't handle verify-caps
 705         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 706                              "400 Bad Request",
 707                              "is not a valid file- or directory- cap",
 708                              self.GET, base)
 709         return d
 710
 711     def test_GET_FILE_URI(self):
 712         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 713         d = self.GET(base)
 714         d.addCallback(self.failUnlessIsBarDotTxt)
 715         return d
 716
 717     def test_GET_FILE_URI_badchild(self):
 718         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 719         errmsg = "Files have no children, certainly not named 'boguschild'"
 720         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 721                              "400 Bad Request", errmsg,
 722                              self.GET, base)
 723         return d
 724
 725     def test_PUT_FILE_URI_badchild(self):
 726         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 727         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 728         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 729                              "400 Bad Request", errmsg,
 730                              self.PUT, base, "")
 731         return d
 732
 733     def test_GET_FILEURL_save(self):
 734         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 735         # TODO: look at the headers, expect a Content-Disposition: attachment
 736         # header.
 737         d.addCallback(self.failUnlessIsBarDotTxt)
 738         return d
 739
 740     def test_GET_FILEURL_missing(self):
 741         d = self.GET(self.public_url + "/foo/missing")
 742         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 743         return d
 744
 745     def test_PUT_NEWFILEURL(self):
 746         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 747         # TODO: we lose the response code, so we can't check this
 748         #self.failUnlessEqual(responsecode, 201)
 749         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 750         d.addCallback(lambda res:
 751                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 752                                                       self.NEWFILE_CONTENTS))
 753         return d
 754
 755     def test_PUT_NEWFILEURL_range_bad(self):
 756         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 757         target = self.public_url + "/foo/new.txt"
 758         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 759                              "501 Not Implemented",
 760                              "Content-Range in PUT not yet supported",
 761                              # (and certainly not for immutable files)
 762                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 763                              headers=headers)
 764         d.addCallback(lambda res:
 765                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 766         return d
 767
 768     def test_PUT_NEWFILEURL_mutable(self):
 769         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 770                      self.NEWFILE_CONTENTS)
 771         # TODO: we lose the response code, so we can't check this
 772         #self.failUnlessEqual(responsecode, 201)
 773         def _check_uri(res):
 774             u = uri.from_string_mutable_filenode(res)
 775             self.failUnless(u.is_mutable())
 776             self.failIf(u.is_readonly())
 777             return res
 778         d.addCallback(_check_uri)
 779         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 780         d.addCallback(lambda res:
 781                       self.failUnlessMutableChildContentsAre(self._foo_node,
 782                                                              u"new.txt",
 783                                                              self.NEWFILE_CONTENTS))
 784         return d
 785
 786     def test_PUT_NEWFILEURL_mutable_toobig(self):
 787         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 788                              "413 Request Entity Too Large",
 789                              "SDMF is limited to one segment, and 10001 > 10000",
 790                              self.PUT,
 791                              self.public_url + "/foo/new.txt?mutable=true",
 792                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 793         return d
 794
 795     def test_PUT_NEWFILEURL_replace(self):
 796         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 797         # TODO: we lose the response code, so we can't check this
 798         #self.failUnlessEqual(responsecode, 200)
 799         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 800         d.addCallback(lambda res:
 801                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 802                                                       self.NEWFILE_CONTENTS))
 803         return d
 804
 805     def test_PUT_NEWFILEURL_bad_t(self):
 806         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 807                              "PUT to a file: bad t=bogus",
 808                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 809                              "contents")
 810         return d
 811
 812     def test_PUT_NEWFILEURL_no_replace(self):
 813         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 814                      self.NEWFILE_CONTENTS)
 815         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 816                   "409 Conflict",
 817                   "There was already a child by that name, and you asked me "
 818                   "to not replace it")
 819         return d
 820
 821     def test_PUT_NEWFILEURL_mkdirs(self):
 822         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 823         fn = self._foo_node
 824         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 825         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 826         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 827         d.addCallback(lambda res:
 828                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 829                                                       self.NEWFILE_CONTENTS))
 830         return d
 831
 832     def test_PUT_NEWFILEURL_blocked(self):
 833         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 834                      self.NEWFILE_CONTENTS)
 835         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 836                   "409 Conflict",
 837                   "Unable to create directory 'blockingfile': a file was in the way")
 838         return d
 839
 840     def test_DELETE_FILEURL(self):
 841         d = self.DELETE(self.public_url + "/foo/bar.txt")
 842         d.addCallback(lambda res:
 843                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 844         return d
 845
 846     def test_DELETE_FILEURL_missing(self):
 847         d = self.DELETE(self.public_url + "/foo/missing")
 848         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 849         return d
 850
 851     def test_DELETE_FILEURL_missing2(self):
 852         d = self.DELETE(self.public_url + "/missing/missing")
 853         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 854         return d
 855
 856     def test_GET_FILEURL_json(self):
 857         # twisted.web.http.parse_qs ignores any query args without an '=', so
 858         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 859         # instead. This may make it tricky to emulate the S3 interface
 860         # completely.
 861         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 862         d.addCallback(self.failUnlessIsBarJSON)
 863         return d
 864
 865     def test_GET_FILEURL_json_missing(self):
 866         d = self.GET(self.public_url + "/foo/missing?json")
 867         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 868         return d
 869
 870     def test_GET_FILEURL_uri(self):
 871         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 872         def _check(res):
 873             self.failUnlessEqual(res, self._bar_txt_uri)
 874         d.addCallback(_check)
 875         d.addCallback(lambda res:
 876                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 877         def _check2(res):
 878             # for now, for files, uris and readonly-uris are the same
 879             self.failUnlessEqual(res, self._bar_txt_uri)
 880         d.addCallback(_check2)
 881         return d
 882
 883     def test_GET_FILEURL_badtype(self):
 884         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 885                                   "bad t=bogus",
 886                                   self.GET,
 887                                   self.public_url + "/foo/bar.txt?t=bogus")
 888         return d
 889
 890     def test_GET_FILEURL_uri_missing(self):
 891         d = self.GET(self.public_url + "/foo/missing?t=uri")
 892         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 893         return d
 894
 895     def test_GET_DIRURL(self):
 896         # the addSlash means we get a redirect here
 897         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 898         ROOT = "../../.."
 899         d = self.GET(self.public_url + "/foo", followRedirect=True)
 900         def _check(res):
 901             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 902                             in res, res)
 903             # the FILE reference points to a URI, but it should end in bar.txt
 904             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 905                        (ROOT, urllib.quote(self._bar_txt_uri)))
 906             get_bar = "".join([r'<td>',
 907                                r'<a href="%s">bar.txt</a>' % bar_url,
 908                                r'</td>',
 909                                r'\s+<td>FILE</td>',
 910                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 911                                ])
 912             self.failUnless(re.search(get_bar, res), res)
 913             for line in res.split("\n"):
 914                 # find the line that contains the delete button for bar.txt
 915                 if ("form action" in line and
 916                     'value="delete"' in line and
 917                     'value="bar.txt"' in line):
 918                     # the form target should use a relative URL
 919                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 920                     self.failUnless(('action="%s"' % foo_url) in line, line)
 921                     # and the when_done= should too
 922                     #done_url = urllib.quote(???)
 923                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 924                     #                in line, line)
 925                     break
 926             else:
 927                 self.fail("unable to find delete-bar.txt line", res)
 928
 929             # the DIR reference just points to a URI
 930             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 931             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 932                        + r'\s+<td>DIR</td>')
 933             self.failUnless(re.search(get_sub, res), res)
 934         d.addCallback(_check)
 935
 936         # look at a directory which is readonly
 937         d.addCallback(lambda res:
 938                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 939         def _check2(res):
 940             self.failUnless("(readonly)" in res, res)
 941             self.failIf("Upload a file" in res, res)
 942         d.addCallback(_check2)
 943
 944         # and at a directory that contains a readonly directory
 945         d.addCallback(lambda res:
 946                       self.GET(self.public_url, followRedirect=True))
 947         def _check3(res):
 948             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 949                                       '</td>\s+<td>DIR-RO</td>', res))
 950         d.addCallback(_check3)
 951
 952         return d
 953
 954     def test_GET_DIRURL_badtype(self):
 955         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 956                                   400, "Bad Request",
 957                                   "bad t=bogus",
 958                                   self.GET,
 959                                   self.public_url + "/foo?t=bogus")
 960         return d
 961
 962     def test_GET_DIRURL_json(self):
 963         d = self.GET(self.public_url + "/foo?t=json")
 964         d.addCallback(self.failUnlessIsFooJSON)
 965         return d
 966
 967
 968     def test_POST_DIRURL_manifest_no_ophandle(self):
 969         d = self.shouldFail2(error.Error,
 970                              "test_POST_DIRURL_manifest_no_ophandle",
 971                              "400 Bad Request",
 972                              "slow operation requires ophandle=",
 973                              self.POST, self.public_url, t="start-manifest")
 974         return d
 975
 976     def test_POST_DIRURL_manifest(self):
 977         d = defer.succeed(None)
 978         def getman(ignored, output):
 979             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 980                           followRedirect=True)
 981             d.addCallback(self.wait_for_operation, "125")
 982             d.addCallback(self.get_operation_results, "125", output)
 983             return d
 984         d.addCallback(getman, None)
 985         def _got_html(manifest):
 986             self.failUnless("Manifest of SI=" in manifest)
 987             self.failUnless("<td>sub</td>" in manifest)
 988             self.failUnless(self._sub_uri in manifest)
 989             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 990         d.addCallback(_got_html)
 991
 992         # both t=status and unadorned GET should be identical
 993         d.addCallback(lambda res: self.GET("/operations/125"))
 994         d.addCallback(_got_html)
 995
 996         d.addCallback(getman, "html")
 997         d.addCallback(_got_html)
 998         d.addCallback(getman, "text")
 999         def _got_text(manifest):
1000             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1001             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1002         d.addCallback(_got_text)
1003         d.addCallback(getman, "JSON")
1004         def _got_json(res):
1005             data = res["manifest"]
1006             got = {}
1007             for (path_list, cap) in data:
1008                 got[tuple(path_list)] = cap
1009             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1010             self.failUnless((u"sub",u"baz.txt") in got)
1011             self.failUnless("finished" in res)
1012             self.failUnless("origin" in res)
1013             self.failUnless("storage-index" in res)
1014             self.failUnless("verifycaps" in res)
1015             self.failUnless("stats" in res)
1016         d.addCallback(_got_json)
1017         return d
1018
1019     def test_POST_DIRURL_deepsize_no_ophandle(self):
1020         d = self.shouldFail2(error.Error,
1021                              "test_POST_DIRURL_deepsize_no_ophandle",
1022                              "400 Bad Request",
1023                              "slow operation requires ophandle=",
1024                              self.POST, self.public_url, t="start-deep-size")
1025         return d
1026
1027     def test_POST_DIRURL_deepsize(self):
1028         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1029                       followRedirect=True)
1030         d.addCallback(self.wait_for_operation, "126")
1031         d.addCallback(self.get_operation_results, "126", "json")
1032         def _got_json(data):
1033             self.failUnlessEqual(data["finished"], True)
1034             size = data["size"]
1035             self.failUnless(size > 1000)
1036         d.addCallback(_got_json)
1037         d.addCallback(self.get_operation_results, "126", "text")
1038         def _got_text(res):
1039             mo = re.search(r'^size: (\d+)$', res, re.M)
1040             self.failUnless(mo, res)
1041             size = int(mo.group(1))
1042             # with directories, the size varies.
1043             self.failUnless(size > 1000)
1044         d.addCallback(_got_text)
1045         return d
1046
1047     def test_POST_DIRURL_deepstats_no_ophandle(self):
1048         d = self.shouldFail2(error.Error,
1049                              "test_POST_DIRURL_deepstats_no_ophandle",
1050                              "400 Bad Request",
1051                              "slow operation requires ophandle=",
1052                              self.POST, self.public_url, t="start-deep-stats")
1053         return d
1054
1055     def test_POST_DIRURL_deepstats(self):
1056         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1057                       followRedirect=True)
1058         d.addCallback(self.wait_for_operation, "127")
1059         d.addCallback(self.get_operation_results, "127", "json")
1060         def _got_json(stats):
1061             expected = {"count-immutable-files": 3,
1062                         "count-mutable-files": 0,
1063                         "count-literal-files": 0,
1064                         "count-files": 3,
1065                         "count-directories": 3,
1066                         "size-immutable-files": 57,
1067                         "size-literal-files": 0,
1068                         #"size-directories": 1912, # varies
1069                         #"largest-directory": 1590,
1070                         "largest-directory-children": 5,
1071                         "largest-immutable-file": 19,
1072                         }
1073             for k,v in expected.iteritems():
1074                 self.failUnlessEqual(stats[k], v,
1075                                      "stats[%s] was %s, not %s" %
1076                                      (k, stats[k], v))
1077             self.failUnlessEqual(stats["size-files-histogram"],
1078                                  [ [11, 31, 3] ])
1079         d.addCallback(_got_json)
1080         return d
1081
1082     def test_POST_DIRURL_stream_manifest(self):
1083         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1084         def _check(res):
1085             self.failUnless(res.endswith("\n"))
1086             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1087             self.failUnlessEqual(len(units), 7)
1088             self.failUnlessEqual(units[-1]["type"], "stats")
1089             first = units[0]
1090             self.failUnlessEqual(first["path"], [])
1091             self.failUnlessEqual(first["cap"], self._foo_uri)
1092             self.failUnlessEqual(first["type"], "directory")
1093             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1094             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1095             self.failIfEqual(baz["storage-index"], None)
1096             self.failIfEqual(baz["verifycap"], None)
1097             self.failIfEqual(baz["repaircap"], None)
1098             return
1099         d.addCallback(_check)
1100         return d
1101
1102     def test_GET_DIRURL_uri(self):
1103         d = self.GET(self.public_url + "/foo?t=uri")
1104         def _check(res):
1105             self.failUnlessEqual(res, self._foo_uri)
1106         d.addCallback(_check)
1107         return d
1108
1109     def test_GET_DIRURL_readonly_uri(self):
1110         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1111         def _check(res):
1112             self.failUnlessEqual(res, self._foo_readonly_uri)
1113         d.addCallback(_check)
1114         return d
1115
1116     def test_PUT_NEWDIRURL(self):
1117         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1118         d.addCallback(lambda res:
1119                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1120         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1121         d.addCallback(self.failUnlessNodeKeysAre, [])
1122         return d
1123
1124     def test_PUT_NEWDIRURL_exists(self):
1125         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1126         d.addCallback(lambda res:
1127                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1128         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1129         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1130         return d
1131
1132     def test_PUT_NEWDIRURL_blocked(self):
1133         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1134                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1135                              self.PUT,
1136                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1137         d.addCallback(lambda res:
1138                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1139         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1140         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1141         return d
1142
1143     def test_PUT_NEWDIRURL_mkdir_p(self):
1144         d = defer.succeed(None)
1145         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1146         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1147         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1148         def mkdir_p(mkpnode):
1149             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1150             d = self.POST(url)
1151             def made_subsub(ssuri):
1152                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1153                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1154                 d = self.POST(url)
1155                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1156                 return d
1157             d.addCallback(made_subsub)
1158             return d
1159         d.addCallback(mkdir_p)
1160         return d
1161
1162     def test_PUT_NEWDIRURL_mkdirs(self):
1163         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1164         d.addCallback(lambda res:
1165                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1166         d.addCallback(lambda res:
1167                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1168         d.addCallback(lambda res:
1169                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1170         d.addCallback(self.failUnlessNodeKeysAre, [])
1171         return d
1172
1173     def test_DELETE_DIRURL(self):
1174         d = self.DELETE(self.public_url + "/foo")
1175         d.addCallback(lambda res:
1176                       self.failIfNodeHasChild(self.public_root, u"foo"))
1177         return d
1178
1179     def test_DELETE_DIRURL_missing(self):
1180         d = self.DELETE(self.public_url + "/foo/missing")
1181         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1182         d.addCallback(lambda res:
1183                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1184         return d
1185
1186     def test_DELETE_DIRURL_missing2(self):
1187         d = self.DELETE(self.public_url + "/missing")
1188         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1189         return d
1190
1191     def dump_root(self):
1192         print "NODEWALK"
1193         w = webish.DirnodeWalkerMixin()
1194         def visitor(childpath, childnode, metadata):
1195             print childpath
1196         d = w.walk(self.public_root, visitor)
1197         return d
1198
1199     def failUnlessNodeKeysAre(self, node, expected_keys):
1200         for k in expected_keys:
1201             assert isinstance(k, unicode)
1202         d = node.list()
1203         def _check(children):
1204             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1205         d.addCallback(_check)
1206         return d
1207     def failUnlessNodeHasChild(self, node, name):
1208         assert isinstance(name, unicode)
1209         d = node.list()
1210         def _check(children):
1211             self.failUnless(name in children)
1212         d.addCallback(_check)
1213         return d
1214     def failIfNodeHasChild(self, node, name):
1215         assert isinstance(name, unicode)
1216         d = node.list()
1217         def _check(children):
1218             self.failIf(name in children)
1219         d.addCallback(_check)
1220         return d
1221
1222     def failUnlessChildContentsAre(self, node, name, expected_contents):
1223         assert isinstance(name, unicode)
1224         d = node.get_child_at_path(name)
1225         d.addCallback(lambda node: node.download_to_data())
1226         def _check(contents):
1227             self.failUnlessEqual(contents, expected_contents)
1228         d.addCallback(_check)
1229         return d
1230
1231     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1232         assert isinstance(name, unicode)
1233         d = node.get_child_at_path(name)
1234         d.addCallback(lambda node: node.download_best_version())
1235         def _check(contents):
1236             self.failUnlessEqual(contents, expected_contents)
1237         d.addCallback(_check)
1238         return d
1239
1240     def failUnlessChildURIIs(self, node, name, expected_uri):
1241         assert isinstance(name, unicode)
1242         d = node.get_child_at_path(name)
1243         def _check(child):
1244             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1245         d.addCallback(_check)
1246         return d
1247
1248     def failUnlessURIMatchesChild(self, got_uri, node, name):
1249         assert isinstance(name, unicode)
1250         d = node.get_child_at_path(name)
1251         def _check(child):
1252             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1253         d.addCallback(_check)
1254         return d
1255
1256     def failUnlessCHKURIHasContents(self, got_uri, contents):
1257         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1258
1259     def test_POST_upload(self):
1260         d = self.POST(self.public_url + "/foo", t="upload",
1261                       file=("new.txt", self.NEWFILE_CONTENTS))
1262         fn = self._foo_node
1263         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1264         d.addCallback(lambda res:
1265                       self.failUnlessChildContentsAre(fn, u"new.txt",
1266                                                       self.NEWFILE_CONTENTS))
1267         return d
1268
1269     def test_POST_upload_unicode(self):
1270         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1271         d = self.POST(self.public_url + "/foo", t="upload",
1272                       file=(filename, self.NEWFILE_CONTENTS))
1273         fn = self._foo_node
1274         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1275         d.addCallback(lambda res:
1276                       self.failUnlessChildContentsAre(fn, filename,
1277                                                       self.NEWFILE_CONTENTS))
1278         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1279         d.addCallback(lambda res: self.GET(target_url))
1280         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1281                                                             self.NEWFILE_CONTENTS,
1282                                                             contents))
1283         return d
1284
1285     def test_POST_upload_unicode_named(self):
1286         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1287         d = self.POST(self.public_url + "/foo", t="upload",
1288                       name=filename,
1289                       file=("overridden", self.NEWFILE_CONTENTS))
1290         fn = self._foo_node
1291         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1292         d.addCallback(lambda res:
1293                       self.failUnlessChildContentsAre(fn, filename,
1294                                                       self.NEWFILE_CONTENTS))
1295         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1296         d.addCallback(lambda res: self.GET(target_url))
1297         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1298                                                             self.NEWFILE_CONTENTS,
1299                                                             contents))
1300         return d
1301
1302     def test_POST_upload_no_link(self):
1303         d = self.POST("/uri", t="upload",
1304                       file=("new.txt", self.NEWFILE_CONTENTS))
1305         def _check_upload_results(page):
1306             # this should be a page which describes the results of the upload
1307             # that just finished.
1308             self.failUnless("Upload Results:" in page)
1309             self.failUnless("URI:" in page)
1310             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1311             mo = uri_re.search(page)
1312             self.failUnless(mo, page)
1313             new_uri = mo.group(1)
1314             return new_uri
1315         d.addCallback(_check_upload_results)
1316         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1317         return d
1318
1319     def test_POST_upload_no_link_whendone(self):
1320         d = self.POST("/uri", t="upload", when_done="/",
1321                       file=("new.txt", self.NEWFILE_CONTENTS))
1322         d.addBoth(self.shouldRedirect, "/")
1323         return d
1324
1325     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1326         d = defer.maybeDeferred(callable, *args, **kwargs)
1327         def done(res):
1328             if isinstance(res, failure.Failure):
1329                 res.trap(error.PageRedirect)
1330                 statuscode = res.value.status
1331                 target = res.value.location
1332                 return checker(statuscode, target)
1333             self.fail("%s: callable was supposed to redirect, not return '%s'"
1334                       % (which, res))
1335         d.addBoth(done)
1336         return d
1337
1338     def test_POST_upload_no_link_whendone_results(self):
1339         def check(statuscode, target):
1340             self.failUnlessEqual(statuscode, str(http.FOUND))
1341             self.failUnless(target.startswith(self.webish_url), target)
1342             return client.getPage(target, method="GET")
1343         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1344                                  check,
1345                                  self.POST, "/uri", t="upload",
1346                                  when_done="/uri/%(uri)s",
1347                                  file=("new.txt", self.NEWFILE_CONTENTS))
1348         d.addCallback(lambda res:
1349                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1350         return d
1351
1352     def test_POST_upload_no_link_mutable(self):
1353         d = self.POST("/uri", t="upload", mutable="true",
1354                       file=("new.txt", self.NEWFILE_CONTENTS))
1355         def _check(new_uri):
1356             new_uri = new_uri.strip()
1357             self.new_uri = new_uri
1358             u = IURI(new_uri)
1359             self.failUnless(IMutableFileURI.providedBy(u))
1360             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1361             n = self.s.create_node_from_uri(new_uri)
1362             return n.download_best_version()
1363         d.addCallback(_check)
1364         def _check2(data):
1365             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1366             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1367         d.addCallback(_check2)
1368         def _check3(data):
1369             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1370             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1371         d.addCallback(_check3)
1372         def _check4(data):
1373             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1374         d.addCallback(_check4)
1375         return d
1376
1377     def test_POST_upload_no_link_mutable_toobig(self):
1378         d = self.shouldFail2(error.Error,
1379                              "test_POST_upload_no_link_mutable_toobig",
1380                              "413 Request Entity Too Large",
1381                              "SDMF is limited to one segment, and 10001 > 10000",
1382                              self.POST,
1383                              "/uri", t="upload", mutable="true",
1384                              file=("new.txt",
1385                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1386         return d
1387
1388     def test_POST_upload_mutable(self):
1389         # this creates a mutable file
1390         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1391                       file=("new.txt", self.NEWFILE_CONTENTS))
1392         fn = self._foo_node
1393         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1394         d.addCallback(lambda res:
1395                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1396                                                              self.NEWFILE_CONTENTS))
1397         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1398         def _got(newnode):
1399             self.failUnless(IMutableFileNode.providedBy(newnode))
1400             self.failUnless(newnode.is_mutable())
1401             self.failIf(newnode.is_readonly())
1402             self._mutable_node = newnode
1403             self._mutable_uri = newnode.get_uri()
1404         d.addCallback(_got)
1405
1406         # now upload it again and make sure that the URI doesn't change
1407         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1408         d.addCallback(lambda res:
1409                       self.POST(self.public_url + "/foo", t="upload",
1410                                 mutable="true",
1411                                 file=("new.txt", NEWER_CONTENTS)))
1412         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1413         d.addCallback(lambda res:
1414                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1415                                                              NEWER_CONTENTS))
1416         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1417         def _got2(newnode):
1418             self.failUnless(IMutableFileNode.providedBy(newnode))
1419             self.failUnless(newnode.is_mutable())
1420             self.failIf(newnode.is_readonly())
1421             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1422         d.addCallback(_got2)
1423
1424         # upload a second time, using PUT instead of POST
1425         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1426         d.addCallback(lambda res:
1427                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1428         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1429         d.addCallback(lambda res:
1430                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1431                                                              NEW2_CONTENTS))
1432
1433         # finally list the directory, since mutable files are displayed
1434         # slightly differently
1435
1436         d.addCallback(lambda res:
1437                       self.GET(self.public_url + "/foo/",
1438                                followRedirect=True))
1439         def _check_page(res):
1440             # TODO: assert more about the contents
1441             self.failUnless("SSK" in res)
1442             return res
1443         d.addCallback(_check_page)
1444
1445         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1446         def _got3(newnode):
1447             self.failUnless(IMutableFileNode.providedBy(newnode))
1448             self.failUnless(newnode.is_mutable())
1449             self.failIf(newnode.is_readonly())
1450             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1451         d.addCallback(_got3)
1452
1453         # look at the JSON form of the enclosing directory
1454         d.addCallback(lambda res:
1455                       self.GET(self.public_url + "/foo/?t=json",
1456                                followRedirect=True))
1457         def _check_page_json(res):
1458             parsed = simplejson.loads(res)
1459             self.failUnlessEqual(parsed[0], "dirnode")
1460             children = dict( [(unicode(name),value)
1461                               for (name,value)
1462                               in parsed[1]["children"].iteritems()] )
1463             self.failUnless("new.txt" in children)
1464             new_json = children["new.txt"]
1465             self.failUnlessEqual(new_json[0], "filenode")
1466             self.failUnless(new_json[1]["mutable"])
1467             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1468             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1469             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1470         d.addCallback(_check_page_json)
1471
1472         # and the JSON form of the file
1473         d.addCallback(lambda res:
1474                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1475         def _check_file_json(res):
1476             parsed = simplejson.loads(res)
1477             self.failUnlessEqual(parsed[0], "filenode")
1478             self.failUnless(parsed[1]["mutable"])
1479             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1480             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1481             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1482         d.addCallback(_check_file_json)
1483
1484         # and look at t=uri and t=readonly-uri
1485         d.addCallback(lambda res:
1486                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1487         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1488         d.addCallback(lambda res:
1489                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1490         def _check_ro_uri(res):
1491             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1492             self.failUnlessEqual(res, ro_uri)
1493         d.addCallback(_check_ro_uri)
1494
1495         # make sure we can get to it from /uri/URI
1496         d.addCallback(lambda res:
1497                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1498         d.addCallback(lambda res:
1499                       self.failUnlessEqual(res, NEW2_CONTENTS))
1500
1501         # and that HEAD computes the size correctly
1502         d.addCallback(lambda res:
1503                       self.HEAD(self.public_url + "/foo/new.txt",
1504                                 return_response=True))
1505         def _got_headers((res, status, headers)):
1506             self.failUnlessEqual(res, "")
1507             self.failUnlessEqual(headers["content-length"][0],
1508                                  str(len(NEW2_CONTENTS)))
1509             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1510         d.addCallback(_got_headers)
1511
1512         # make sure that size errors are displayed correctly for overwrite
1513         d.addCallback(lambda res:
1514                       self.shouldFail2(error.Error,
1515                                        "test_POST_upload_mutable-toobig",
1516                                        "413 Request Entity Too Large",
1517                                        "SDMF is limited to one segment, and 10001 > 10000",
1518                                        self.POST,
1519                                        self.public_url + "/foo", t="upload",
1520                                        mutable="true",
1521                                        file=("new.txt",
1522                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1523                                        ))
1524
1525         d.addErrback(self.dump_error)
1526         return d
1527
1528     def test_POST_upload_mutable_toobig(self):
1529         d = self.shouldFail2(error.Error,
1530                              "test_POST_upload_no_link_mutable_toobig",
1531                              "413 Request Entity Too Large",
1532                              "SDMF is limited to one segment, and 10001 > 10000",
1533                              self.POST,
1534                              self.public_url + "/foo",
1535                              t="upload", mutable="true",
1536                              file=("new.txt",
1537                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1538         return d
1539
1540     def dump_error(self, f):
1541         # if the web server returns an error code (like 400 Bad Request),
1542         # web.client.getPage puts the HTTP response body into the .response
1543         # attribute of the exception object that it gives back. It does not
1544         # appear in the Failure's repr(), so the ERROR that trial displays
1545         # will be rather terse and unhelpful. addErrback this method to the
1546         # end of your chain to get more information out of these errors.
1547         if f.check(error.Error):
1548             print "web.error.Error:"
1549             print f
1550             print f.value.response
1551         return f
1552
1553     def test_POST_upload_replace(self):
1554         d = self.POST(self.public_url + "/foo", t="upload",
1555                       file=("bar.txt", self.NEWFILE_CONTENTS))
1556         fn = self._foo_node
1557         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1558         d.addCallback(lambda res:
1559                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1560                                                       self.NEWFILE_CONTENTS))
1561         return d
1562
1563     def test_POST_upload_no_replace_ok(self):
1564         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1565                       file=("new.txt", self.NEWFILE_CONTENTS))
1566         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1567         d.addCallback(lambda res: self.failUnlessEqual(res,
1568                                                        self.NEWFILE_CONTENTS))
1569         return d
1570
1571     def test_POST_upload_no_replace_queryarg(self):
1572         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1573                       file=("bar.txt", self.NEWFILE_CONTENTS))
1574         d.addBoth(self.shouldFail, error.Error,
1575                   "POST_upload_no_replace_queryarg",
1576                   "409 Conflict",
1577                   "There was already a child by that name, and you asked me "
1578                   "to not replace it")
1579         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1580         d.addCallback(self.failUnlessIsBarDotTxt)
1581         return d
1582
1583     def test_POST_upload_no_replace_field(self):
1584         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1585                       file=("bar.txt", self.NEWFILE_CONTENTS))
1586         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1587                   "409 Conflict",
1588                   "There was already a child by that name, and you asked me "
1589                   "to not replace it")
1590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1591         d.addCallback(self.failUnlessIsBarDotTxt)
1592         return d
1593
1594     def test_POST_upload_whendone(self):
1595         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1596                       file=("new.txt", self.NEWFILE_CONTENTS))
1597         d.addBoth(self.shouldRedirect, "/THERE")
1598         fn = self._foo_node
1599         d.addCallback(lambda res:
1600                       self.failUnlessChildContentsAre(fn, u"new.txt",
1601                                                       self.NEWFILE_CONTENTS))
1602         return d
1603
1604     def test_POST_upload_named(self):
1605         fn = self._foo_node
1606         d = self.POST(self.public_url + "/foo", t="upload",
1607                       name="new.txt", file=self.NEWFILE_CONTENTS)
1608         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1609         d.addCallback(lambda res:
1610                       self.failUnlessChildContentsAre(fn, u"new.txt",
1611                                                       self.NEWFILE_CONTENTS))
1612         return d
1613
1614     def test_POST_upload_named_badfilename(self):
1615         d = self.POST(self.public_url + "/foo", t="upload",
1616                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1617         d.addBoth(self.shouldFail, error.Error,
1618                   "test_POST_upload_named_badfilename",
1619                   "400 Bad Request",
1620                   "name= may not contain a slash",
1621                   )
1622         # make sure that nothing was added
1623         d.addCallback(lambda res:
1624                       self.failUnlessNodeKeysAre(self._foo_node,
1625                                                  [u"bar.txt", u"blockingfile",
1626                                                   u"empty", u"n\u00fc.txt",
1627                                                   u"sub"]))
1628         return d
1629
1630     def test_POST_FILEURL_check(self):
1631         bar_url = self.public_url + "/foo/bar.txt"
1632         d = self.POST(bar_url, t="check")
1633         def _check(res):
1634             self.failUnless("Healthy :" in res)
1635         d.addCallback(_check)
1636         redir_url = "http://allmydata.org/TARGET"
1637         def _check2(statuscode, target):
1638             self.failUnlessEqual(statuscode, str(http.FOUND))
1639             self.failUnlessEqual(target, redir_url)
1640         d.addCallback(lambda res:
1641                       self.shouldRedirect2("test_POST_FILEURL_check",
1642                                            _check2,
1643                                            self.POST, bar_url,
1644                                            t="check",
1645                                            when_done=redir_url))
1646         d.addCallback(lambda res:
1647                       self.POST(bar_url, t="check", return_to=redir_url))
1648         def _check3(res):
1649             self.failUnless("Healthy :" in res)
1650             self.failUnless("Return to parent directory" in res)
1651             self.failUnless(redir_url in res)
1652         d.addCallback(_check3)
1653
1654         d.addCallback(lambda res:
1655                       self.POST(bar_url, t="check", output="JSON"))
1656         def _check_json(res):
1657             data = simplejson.loads(res)
1658             self.failUnless("storage-index" in data)
1659             self.failUnless(data["results"]["healthy"])
1660         d.addCallback(_check_json)
1661
1662         return d
1663
1664     def test_POST_FILEURL_check_and_repair(self):
1665         bar_url = self.public_url + "/foo/bar.txt"
1666         d = self.POST(bar_url, t="check", repair="true")
1667         def _check(res):
1668             self.failUnless("Healthy :" in res)
1669         d.addCallback(_check)
1670         redir_url = "http://allmydata.org/TARGET"
1671         def _check2(statuscode, target):
1672             self.failUnlessEqual(statuscode, str(http.FOUND))
1673             self.failUnlessEqual(target, redir_url)
1674         d.addCallback(lambda res:
1675                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1676                                            _check2,
1677                                            self.POST, bar_url,
1678                                            t="check", repair="true",
1679                                            when_done=redir_url))
1680         d.addCallback(lambda res:
1681                       self.POST(bar_url, t="check", return_to=redir_url))
1682         def _check3(res):
1683             self.failUnless("Healthy :" in res)
1684             self.failUnless("Return to parent directory" in res)
1685             self.failUnless(redir_url in res)
1686         d.addCallback(_check3)
1687         return d
1688
1689     def test_POST_DIRURL_check(self):
1690         foo_url = self.public_url + "/foo/"
1691         d = self.POST(foo_url, t="check")
1692         def _check(res):
1693             self.failUnless("Healthy :" in res, res)
1694         d.addCallback(_check)
1695         redir_url = "http://allmydata.org/TARGET"
1696         def _check2(statuscode, target):
1697             self.failUnlessEqual(statuscode, str(http.FOUND))
1698             self.failUnlessEqual(target, redir_url)
1699         d.addCallback(lambda res:
1700                       self.shouldRedirect2("test_POST_DIRURL_check",
1701                                            _check2,
1702                                            self.POST, foo_url,
1703                                            t="check",
1704                                            when_done=redir_url))
1705         d.addCallback(lambda res:
1706                       self.POST(foo_url, t="check", return_to=redir_url))
1707         def _check3(res):
1708             self.failUnless("Healthy :" in res, res)
1709             self.failUnless("Return to parent directory" in res)
1710             self.failUnless(redir_url in res)
1711         d.addCallback(_check3)
1712
1713         d.addCallback(lambda res:
1714                       self.POST(foo_url, t="check", output="JSON"))
1715         def _check_json(res):
1716             data = simplejson.loads(res)
1717             self.failUnless("storage-index" in data)
1718             self.failUnless(data["results"]["healthy"])
1719         d.addCallback(_check_json)
1720
1721         return d
1722
1723     def test_POST_DIRURL_check_and_repair(self):
1724         foo_url = self.public_url + "/foo/"
1725         d = self.POST(foo_url, t="check", repair="true")
1726         def _check(res):
1727             self.failUnless("Healthy :" in res, res)
1728         d.addCallback(_check)
1729         redir_url = "http://allmydata.org/TARGET"
1730         def _check2(statuscode, target):
1731             self.failUnlessEqual(statuscode, str(http.FOUND))
1732             self.failUnlessEqual(target, redir_url)
1733         d.addCallback(lambda res:
1734                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1735                                            _check2,
1736                                            self.POST, foo_url,
1737                                            t="check", repair="true",
1738                                            when_done=redir_url))
1739         d.addCallback(lambda res:
1740                       self.POST(foo_url, t="check", return_to=redir_url))
1741         def _check3(res):
1742             self.failUnless("Healthy :" in res)
1743             self.failUnless("Return to parent directory" in res)
1744             self.failUnless(redir_url in res)
1745         d.addCallback(_check3)
1746         return d
1747
1748     def wait_for_operation(self, ignored, ophandle):
1749         url = "/operations/" + ophandle
1750         url += "?t=status&output=JSON"
1751         d = self.GET(url)
1752         def _got(res):
1753             data = simplejson.loads(res)
1754             if not data["finished"]:
1755                 d = self.stall(delay=1.0)
1756                 d.addCallback(self.wait_for_operation, ophandle)
1757                 return d
1758             return data
1759         d.addCallback(_got)
1760         return d
1761
1762     def get_operation_results(self, ignored, ophandle, output=None):
1763         url = "/operations/" + ophandle
1764         url += "?t=status"
1765         if output:
1766             url += "&output=" + output
1767         d = self.GET(url)
1768         def _got(res):
1769             if output and output.lower() == "json":
1770                 return simplejson.loads(res)
1771             return res
1772         d.addCallback(_got)
1773         return d
1774
1775     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1776         d = self.shouldFail2(error.Error,
1777                              "test_POST_DIRURL_deepcheck_no_ophandle",
1778                              "400 Bad Request",
1779                              "slow operation requires ophandle=",
1780                              self.POST, self.public_url, t="start-deep-check")
1781         return d
1782
1783     def test_POST_DIRURL_deepcheck(self):
1784         def _check_redirect(statuscode, target):
1785             self.failUnlessEqual(statuscode, str(http.FOUND))
1786             self.failUnless(target.endswith("/operations/123"))
1787         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1788                                  self.POST, self.public_url,
1789                                  t="start-deep-check", ophandle="123")
1790         d.addCallback(self.wait_for_operation, "123")
1791         def _check_json(data):
1792             self.failUnlessEqual(data["finished"], True)
1793             self.failUnlessEqual(data["count-objects-checked"], 8)
1794             self.failUnlessEqual(data["count-objects-healthy"], 8)
1795         d.addCallback(_check_json)
1796         d.addCallback(self.get_operation_results, "123", "html")
1797         def _check_html(res):
1798             self.failUnless("Objects Checked: <span>8</span>" in res)
1799             self.failUnless("Objects Healthy: <span>8</span>" in res)
1800         d.addCallback(_check_html)
1801
1802         d.addCallback(lambda res:
1803                       self.GET("/operations/123/"))
1804         d.addCallback(_check_html) # should be the same as without the slash
1805
1806         d.addCallback(lambda res:
1807                       self.shouldFail2(error.Error, "one", "404 Not Found",
1808                                        "No detailed results for SI bogus",
1809                                        self.GET, "/operations/123/bogus"))
1810
1811         foo_si = self._foo_node.get_storage_index()
1812         foo_si_s = base32.b2a(foo_si)
1813         d.addCallback(lambda res:
1814                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1815         def _check_foo_json(res):
1816             data = simplejson.loads(res)
1817             self.failUnlessEqual(data["storage-index"], foo_si_s)
1818             self.failUnless(data["results"]["healthy"])
1819         d.addCallback(_check_foo_json)
1820         return d
1821
1822     def test_POST_DIRURL_deepcheck_and_repair(self):
1823         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1824                       ophandle="124", output="json", followRedirect=True)
1825         d.addCallback(self.wait_for_operation, "124")
1826         def _check_json(data):
1827             self.failUnlessEqual(data["finished"], True)
1828             self.failUnlessEqual(data["count-objects-checked"], 8)
1829             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1830             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1831             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1832             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1833             self.failUnlessEqual(data["count-repairs-successful"], 0)
1834             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1835             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1836             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1837             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1838         d.addCallback(_check_json)
1839         d.addCallback(self.get_operation_results, "124", "html")
1840         def _check_html(res):
1841             self.failUnless("Objects Checked: <span>8</span>" in res)
1842
1843             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1844             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1845             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1846
1847             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1848             self.failUnless("Repairs Successful: <span>0</span>" in res)
1849             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1850
1851             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1852             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1853             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1854         d.addCallback(_check_html)
1855         return d
1856
1857     def test_POST_FILEURL_bad_t(self):
1858         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1859                              "POST to file: bad t=bogus",
1860                              self.POST, self.public_url + "/foo/bar.txt",
1861                              t="bogus")
1862         return d
1863
1864     def test_POST_mkdir(self): # return value?
1865         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1866         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1867         d.addCallback(self.failUnlessNodeKeysAre, [])
1868         return d
1869
1870     def test_POST_mkdir_2(self):
1871         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1872         d.addCallback(lambda res:
1873                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1874         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875         d.addCallback(self.failUnlessNodeKeysAre, [])
1876         return d
1877
1878     def test_POST_mkdirs_2(self):
1879         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1880         d.addCallback(lambda res:
1881                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1882         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1883         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1884         d.addCallback(self.failUnlessNodeKeysAre, [])
1885         return d
1886
1887     def test_POST_mkdir_no_parentdir_noredirect(self):
1888         d = self.POST("/uri?t=mkdir")
1889         def _after_mkdir(res):
1890             uri.NewDirectoryURI.init_from_string(res)
1891         d.addCallback(_after_mkdir)
1892         return d
1893
1894     def test_POST_mkdir_no_parentdir_redirect(self):
1895         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1896         d.addBoth(self.shouldRedirect, None, statuscode='303')
1897         def _check_target(target):
1898             target = urllib.unquote(target)
1899             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1900         d.addCallback(_check_target)
1901         return d
1902
1903     def test_POST_noparent_bad(self):
1904         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1905                                   "/uri accepts only PUT, PUT?t=mkdir, "
1906                                   "POST?t=upload, and POST?t=mkdir",
1907                                   self.POST, "/uri?t=bogus")
1908         return d
1909
1910     def test_welcome_page_mkdir_button(self):
1911         # Fetch the welcome page.
1912         d = self.GET("/")
1913         def _after_get_welcome_page(res):
1914             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1915             mo = MKDIR_BUTTON_RE.search(res)
1916             formaction = mo.group(1)
1917             formt = mo.group(2)
1918             formaname = mo.group(3)
1919             formavalue = mo.group(4)
1920             return (formaction, formt, formaname, formavalue)
1921         d.addCallback(_after_get_welcome_page)
1922         def _after_parse_form(res):
1923             (formaction, formt, formaname, formavalue) = res
1924             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1925         d.addCallback(_after_parse_form)
1926         d.addBoth(self.shouldRedirect, None, statuscode='303')
1927         return d
1928
1929     def test_POST_mkdir_replace(self): # return value?
1930         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1931         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1932         d.addCallback(self.failUnlessNodeKeysAre, [])
1933         return d
1934
1935     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1936         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1937         d.addBoth(self.shouldFail, error.Error,
1938                   "POST_mkdir_no_replace_queryarg",
1939                   "409 Conflict",
1940                   "There was already a child by that name, and you asked me "
1941                   "to not replace it")
1942         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1943         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1944         return d
1945
1946     def test_POST_mkdir_no_replace_field(self): # return value?
1947         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1948                       replace="false")
1949         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1950                   "409 Conflict",
1951                   "There was already a child by that name, and you asked me "
1952                   "to not replace it")
1953         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1954         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1955         return d
1956
1957     def test_POST_mkdir_whendone_field(self):
1958         d = self.POST(self.public_url + "/foo",
1959                       t="mkdir", name="newdir", when_done="/THERE")
1960         d.addBoth(self.shouldRedirect, "/THERE")
1961         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1962         d.addCallback(self.failUnlessNodeKeysAre, [])
1963         return d
1964
1965     def test_POST_mkdir_whendone_queryarg(self):
1966         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1967                       t="mkdir", name="newdir")
1968         d.addBoth(self.shouldRedirect, "/THERE")
1969         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1970         d.addCallback(self.failUnlessNodeKeysAre, [])
1971         return d
1972
1973     def test_POST_bad_t(self):
1974         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1975                              "POST to a directory with bad t=BOGUS",
1976                              self.POST, self.public_url + "/foo", t="BOGUS")
1977         return d
1978
1979     def test_POST_set_children(self):
1980         contents9, n9, newuri9 = self.makefile(9)
1981         contents10, n10, newuri10 = self.makefile(10)
1982         contents11, n11, newuri11 = self.makefile(11)
1983
1984         reqbody = """{
1985                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1986                                                 "size": 0,
1987                                                 "metadata": {
1988                                                   "ctime": 1002777696.7564139,
1989                                                   "mtime": 1002777696.7564139
1990                                                  }
1991                                                } ],
1992                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1993                                                 "size": 1,
1994                                                 "metadata": {
1995                                                   "ctime": 1002777696.7564139,
1996                                                   "mtime": 1002777696.7564139
1997                                                  }
1998                                                } ],
1999                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2000                                                 "size": 2,
2001                                                 "metadata": {
2002                                                   "ctime": 1002777696.7564139,
2003                                                   "mtime": 1002777696.7564139
2004                                                  }
2005                                                } ]
2006                     }""" % (newuri9, newuri10, newuri11)
2007
2008         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2009
2010         d = client.getPage(url, method="POST", postdata=reqbody)
2011         def _then(res):
2012             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2013             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2014             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2015
2016         d.addCallback(_then)
2017         d.addErrback(self.dump_error)
2018         return d
2019
2020     def test_POST_put_uri(self):
2021         contents, n, newuri = self.makefile(8)
2022         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2023         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2024         d.addCallback(lambda res:
2025                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2026                                                       contents))
2027         return d
2028
2029     def test_POST_put_uri_replace(self):
2030         contents, n, newuri = self.makefile(8)
2031         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2032         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2033         d.addCallback(lambda res:
2034                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2035                                                       contents))
2036         return d
2037
2038     def test_POST_put_uri_no_replace_queryarg(self):
2039         contents, n, newuri = self.makefile(8)
2040         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2041                       name="bar.txt", uri=newuri)
2042         d.addBoth(self.shouldFail, error.Error,
2043                   "POST_put_uri_no_replace_queryarg",
2044                   "409 Conflict",
2045                   "There was already a child by that name, and you asked me "
2046                   "to not replace it")
2047         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2048         d.addCallback(self.failUnlessIsBarDotTxt)
2049         return d
2050
2051     def test_POST_put_uri_no_replace_field(self):
2052         contents, n, newuri = self.makefile(8)
2053         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2054                       name="bar.txt", uri=newuri)
2055         d.addBoth(self.shouldFail, error.Error,
2056                   "POST_put_uri_no_replace_field",
2057                   "409 Conflict",
2058                   "There was already a child by that name, and you asked me "
2059                   "to not replace it")
2060         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2061         d.addCallback(self.failUnlessIsBarDotTxt)
2062         return d
2063
2064     def test_POST_delete(self):
2065         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2066         d.addCallback(lambda res: self._foo_node.list())
2067         def _check(children):
2068             self.failIf(u"bar.txt" in children)
2069         d.addCallback(_check)
2070         return d
2071
2072     def test_POST_rename_file(self):
2073         d = self.POST(self.public_url + "/foo", t="rename",
2074                       from_name="bar.txt", to_name='wibble.txt')
2075         d.addCallback(lambda res:
2076                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2077         d.addCallback(lambda res:
2078                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2079         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2080         d.addCallback(self.failUnlessIsBarDotTxt)
2081         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2082         d.addCallback(self.failUnlessIsBarJSON)
2083         return d
2084
2085     def test_POST_rename_file_redundant(self):
2086         d = self.POST(self.public_url + "/foo", t="rename",
2087                       from_name="bar.txt", to_name='bar.txt')
2088         d.addCallback(lambda res:
2089                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2090         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2091         d.addCallback(self.failUnlessIsBarDotTxt)
2092         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2093         d.addCallback(self.failUnlessIsBarJSON)
2094         return d
2095
2096     def test_POST_rename_file_replace(self):
2097         # rename a file and replace a directory with it
2098         d = self.POST(self.public_url + "/foo", t="rename",
2099                       from_name="bar.txt", to_name='empty')
2100         d.addCallback(lambda res:
2101                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2102         d.addCallback(lambda res:
2103                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2104         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2105         d.addCallback(self.failUnlessIsBarDotTxt)
2106         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2107         d.addCallback(self.failUnlessIsBarJSON)
2108         return d
2109
2110     def test_POST_rename_file_no_replace_queryarg(self):
2111         # rename a file and replace a directory with it
2112         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2113                       from_name="bar.txt", to_name='empty')
2114         d.addBoth(self.shouldFail, error.Error,
2115                   "POST_rename_file_no_replace_queryarg",
2116                   "409 Conflict",
2117                   "There was already a child by that name, and you asked me "
2118                   "to not replace it")
2119         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2120         d.addCallback(self.failUnlessIsEmptyJSON)
2121         return d
2122
2123     def test_POST_rename_file_no_replace_field(self):
2124         # rename a file and replace a directory with it
2125         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2126                       from_name="bar.txt", to_name='empty')
2127         d.addBoth(self.shouldFail, error.Error,
2128                   "POST_rename_file_no_replace_field",
2129                   "409 Conflict",
2130                   "There was already a child by that name, and you asked me "
2131                   "to not replace it")
2132         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2133         d.addCallback(self.failUnlessIsEmptyJSON)
2134         return d
2135
2136     def failUnlessIsEmptyJSON(self, res):
2137         data = simplejson.loads(res)
2138         self.failUnlessEqual(data[0], "dirnode", data)
2139         self.failUnlessEqual(len(data[1]["children"]), 0)
2140
2141     def test_POST_rename_file_slash_fail(self):
2142         d = self.POST(self.public_url + "/foo", t="rename",
2143                       from_name="bar.txt", to_name='kirk/spock.txt')
2144         d.addBoth(self.shouldFail, error.Error,
2145                   "test_POST_rename_file_slash_fail",
2146                   "400 Bad Request",
2147                   "to_name= may not contain a slash",
2148                   )
2149         d.addCallback(lambda res:
2150                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2151         return d
2152
2153     def test_POST_rename_dir(self):
2154         d = self.POST(self.public_url, t="rename",
2155                       from_name="foo", to_name='plunk')
2156         d.addCallback(lambda res:
2157                       self.failIfNodeHasChild(self.public_root, u"foo"))
2158         d.addCallback(lambda res:
2159                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2160         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2161         d.addCallback(self.failUnlessIsFooJSON)
2162         return d
2163
2164     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2165         """ If target is not None then the redirection has to go to target.  If
2166         statuscode is not None then the redirection has to be accomplished with
2167         that HTTP status code."""
2168         if not isinstance(res, failure.Failure):
2169             to_where = (target is None) and "somewhere" or ("to " + target)
2170             self.fail("%s: we were expecting to get redirected %s, not get an"
2171                       " actual page: %s" % (which, to_where, res))
2172         res.trap(error.PageRedirect)
2173         if statuscode is not None:
2174             self.failUnlessEqual(res.value.status, statuscode,
2175                                  "%s: not a redirect" % which)
2176         if target is not None:
2177             # the PageRedirect does not seem to capture the uri= query arg
2178             # properly, so we can't check for it.
2179             realtarget = self.webish_url + target
2180             self.failUnlessEqual(res.value.location, realtarget,
2181                                  "%s: wrong target" % which)
2182         return res.value.location
2183
2184     def test_GET_URI_form(self):
2185         base = "/uri?uri=%s" % self._bar_txt_uri
2186         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2187         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2188         d = self.GET(base)
2189         d.addBoth(self.shouldRedirect, targetbase)
2190         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2191         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2192         d.addCallback(lambda res: self.GET(base+"&t=json"))
2193         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2194         d.addCallback(self.log, "about to get file by uri")
2195         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2196         d.addCallback(self.failUnlessIsBarDotTxt)
2197         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2198         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2199                                            followRedirect=True))
2200         d.addCallback(self.failUnlessIsFooJSON)
2201         d.addCallback(self.log, "got dir by uri")
2202
2203         return d
2204
2205     def test_GET_URI_form_bad(self):
2206         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2207                              "400 Bad Request", "GET /uri requires uri=",
2208                              self.GET, "/uri")
2209         return d
2210
2211     def test_GET_rename_form(self):
2212         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2213                      followRedirect=True)
2214         def _check(res):
2215             self.failUnless('name="when_done" value="."' in res, res)
2216             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2217         d.addCallback(_check)
2218         return d
2219
2220     def log(self, res, msg):
2221         #print "MSG: %s  RES: %s" % (msg, res)
2222         log.msg(msg)
2223         return res
2224
2225     def test_GET_URI_URL(self):
2226         base = "/uri/%s" % self._bar_txt_uri
2227         d = self.GET(base)
2228         d.addCallback(self.failUnlessIsBarDotTxt)
2229         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2230         d.addCallback(self.failUnlessIsBarDotTxt)
2231         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2232         d.addCallback(self.failUnlessIsBarDotTxt)
2233         return d
2234
2235     def test_GET_URI_URL_dir(self):
2236         base = "/uri/%s?t=json" % self._foo_uri
2237         d = self.GET(base)
2238         d.addCallback(self.failUnlessIsFooJSON)
2239         return d
2240
2241     def test_GET_URI_URL_missing(self):
2242         base = "/uri/%s" % self._bad_file_uri
2243         d = self.GET(base)
2244         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2245                   http.GONE, response_substring="NotEnoughSharesError")
2246         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2247         # here? we must arrange for a download to fail after target.open()
2248         # has been called, and then inspect the response to see that it is
2249         # shorter than we expected.
2250         return d
2251
2252     def test_PUT_DIRURL_uri(self):
2253         d = self.s.create_empty_dirnode()
2254         def _made_dir(dn):
2255             new_uri = dn.get_uri()
2256             # replace /foo with a new (empty) directory
2257             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2258             d.addCallback(lambda res:
2259                           self.failUnlessEqual(res.strip(), new_uri))
2260             d.addCallback(lambda res:
2261                           self.failUnlessChildURIIs(self.public_root,
2262                                                     u"foo",
2263                                                     new_uri))
2264             return d
2265         d.addCallback(_made_dir)
2266         return d
2267
2268     def test_PUT_DIRURL_uri_noreplace(self):
2269         d = self.s.create_empty_dirnode()
2270         def _made_dir(dn):
2271             new_uri = dn.get_uri()
2272             # replace /foo with a new (empty) directory, but ask that
2273             # replace=false, so it should fail
2274             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2275                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2276                                  self.PUT,
2277                                  self.public_url + "/foo?t=uri&replace=false",
2278                                  new_uri)
2279             d.addCallback(lambda res:
2280                           self.failUnlessChildURIIs(self.public_root,
2281                                                     u"foo",
2282                                                     self._foo_uri))
2283             return d
2284         d.addCallback(_made_dir)
2285         return d
2286
2287     def test_PUT_DIRURL_bad_t(self):
2288         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2289                                  "400 Bad Request", "PUT to a directory",
2290                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2291         d.addCallback(lambda res:
2292                       self.failUnlessChildURIIs(self.public_root,
2293                                                 u"foo",
2294                                                 self._foo_uri))
2295         return d
2296
2297     def test_PUT_NEWFILEURL_uri(self):
2298         contents, n, new_uri = self.makefile(8)
2299         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2300         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2301         d.addCallback(lambda res:
2302                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2303                                                       contents))
2304         return d
2305
2306     def test_PUT_NEWFILEURL_uri_replace(self):
2307         contents, n, new_uri = self.makefile(8)
2308         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2309         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2310         d.addCallback(lambda res:
2311                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2312                                                       contents))
2313         return d
2314
2315     def test_PUT_NEWFILEURL_uri_no_replace(self):
2316         contents, n, new_uri = self.makefile(8)
2317         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2318         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2319                   "409 Conflict",
2320                   "There was already a child by that name, and you asked me "
2321                   "to not replace it")
2322         return d
2323
2324     def test_PUT_NEWFILE_URI(self):
2325         file_contents = "New file contents here\n"
2326         d = self.PUT("/uri", file_contents)
2327         def _check(uri):
2328             assert isinstance(uri, str), uri
2329             self.failUnless(uri in FakeCHKFileNode.all_contents)
2330             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2331                                  file_contents)
2332             return self.GET("/uri/%s" % uri)
2333         d.addCallback(_check)
2334         def _check2(res):
2335             self.failUnlessEqual(res, file_contents)
2336         d.addCallback(_check2)
2337         return d
2338
2339     def test_PUT_NEWFILE_URI_only_PUT(self):
2340         d = self.PUT("/uri?t=bogus", "")
2341         d.addBoth(self.shouldFail, error.Error,
2342                   "PUT_NEWFILE_URI_only_PUT",
2343                   "400 Bad Request",
2344                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2345         return d
2346
2347     def test_PUT_NEWFILE_URI_mutable(self):
2348         file_contents = "New file contents here\n"
2349         d = self.PUT("/uri?mutable=true", file_contents)
2350         def _check_mutable(uri):
2351             uri = uri.strip()
2352             u = IURI(uri)
2353             self.failUnless(IMutableFileURI.providedBy(u))
2354             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2355             n = self.s.create_node_from_uri(uri)
2356             return n.download_best_version()
2357         d.addCallback(_check_mutable)
2358         def _check2_mutable(data):
2359             self.failUnlessEqual(data, file_contents)
2360         d.addCallback(_check2_mutable)
2361         return d
2362
2363         def _check(uri):
2364             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2365             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2366                                  file_contents)
2367             return self.GET("/uri/%s" % uri)
2368         d.addCallback(_check)
2369         def _check2(res):
2370             self.failUnlessEqual(res, file_contents)
2371         d.addCallback(_check2)
2372         return d
2373
2374     def test_PUT_mkdir(self):
2375         d = self.PUT("/uri?t=mkdir", "")
2376         def _check(uri):
2377             n = self.s.create_node_from_uri(uri.strip())
2378             d2 = self.failUnlessNodeKeysAre(n, [])
2379             d2.addCallback(lambda res:
2380                            self.GET("/uri/%s?t=json" % uri))
2381             return d2
2382         d.addCallback(_check)
2383         d.addCallback(self.failUnlessIsEmptyJSON)
2384         return d
2385
2386     def test_POST_check(self):
2387         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2388         def _done(res):
2389             # this returns a string form of the results, which are probably
2390             # None since we're using fake filenodes.
2391             # TODO: verify that the check actually happened, by changing
2392             # FakeCHKFileNode to count how many times .check() has been
2393             # called.
2394             pass
2395         d.addCallback(_done)
2396         return d
2397
2398     def test_bad_method(self):
2399         url = self.webish_url + self.public_url + "/foo/bar.txt"
2400         d = self.shouldHTTPError2("test_bad_method",
2401                                   501, "Not Implemented",
2402                                   "I don't know how to treat a BOGUS request.",
2403                                   client.getPage, url, method="BOGUS")
2404         return d
2405
2406     def test_short_url(self):
2407         url = self.webish_url + "/uri"
2408         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2409                                   "I don't know how to treat a DELETE request.",
2410                                   client.getPage, url, method="DELETE")
2411         return d
2412
2413     def test_ophandle_bad(self):
2414         url = self.webish_url + "/operations/bogus?t=status"
2415         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2416                                   "unknown/expired handle 'bogus'",
2417                                   client.getPage, url)
2418         return d
2419
2420     def test_ophandle_cancel(self):
2421         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2422                       followRedirect=True)
2423         d.addCallback(lambda ignored:
2424                       self.GET("/operations/128?t=status&output=JSON"))
2425         def _check1(res):
2426             data = simplejson.loads(res)
2427             self.failUnless("finished" in data, res)
2428             monitor = self.ws.root.child_operations.handles["128"][0]
2429             d = self.POST("/operations/128?t=cancel&output=JSON")
2430             def _check2(res):
2431                 data = simplejson.loads(res)
2432                 self.failUnless("finished" in data, res)
2433                 # t=cancel causes the handle to be forgotten
2434                 self.failUnless(monitor.is_cancelled())
2435             d.addCallback(_check2)
2436             return d
2437         d.addCallback(_check1)
2438         d.addCallback(lambda ignored:
2439                       self.shouldHTTPError2("test_ophandle_cancel",
2440                                             404, "404 Not Found",
2441                                             "unknown/expired handle '128'",
2442                                             self.GET,
2443                                             "/operations/128?t=status&output=JSON"))
2444         return d
2445
2446     def test_ophandle_retainfor(self):
2447         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2448                       followRedirect=True)
2449         d.addCallback(lambda ignored:
2450                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2451         def _check1(res):
2452             data = simplejson.loads(res)
2453             self.failUnless("finished" in data, res)
2454         d.addCallback(_check1)
2455         # the retain-for=0 will cause the handle to be expired very soon
2456         d.addCallback(self.stall, 2.0)
2457         d.addCallback(lambda ignored:
2458                       self.shouldHTTPError2("test_ophandle_retainfor",
2459                                             404, "404 Not Found",
2460                                             "unknown/expired handle '129'",
2461                                             self.GET,
2462                                             "/operations/129?t=status&output=JSON"))
2463         return d
2464
2465     def test_ophandle_release_after_complete(self):
2466         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2467                       followRedirect=True)
2468         d.addCallback(self.wait_for_operation, "130")
2469         d.addCallback(lambda ignored:
2470                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2471         # the release-after-complete=true will cause the handle to be expired
2472         d.addCallback(lambda ignored:
2473                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2474                                             404, "404 Not Found",
2475                                             "unknown/expired handle '130'",
2476                                             self.GET,
2477                                             "/operations/130?t=status&output=JSON"))
2478         return d
2479
2480     def test_incident(self):
2481         d = self.POST("/report_incident", details="eek")
2482         def _done(res):
2483             self.failUnless("Thank you for your report!" in res, res)
2484         d.addCallback(_done)
2485         return d
2486
2487     def test_static(self):
2488         webdir = os.path.join(self.staticdir, "subdir")
2489         fileutil.make_dirs(webdir)
2490         f = open(os.path.join(webdir, "hello.txt"), "wb")
2491         f.write("hello")
2492         f.close()
2493
2494         d = self.GET("/static/subdir/hello.txt")
2495         def _check(res):
2496             self.failUnlessEqual(res, "hello")
2497         d.addCallback(_check)
2498         return d
2499
2500
2501 class Util(unittest.TestCase):
2502     def test_abbreviate_time(self):
2503         self.failUnlessEqual(common.abbreviate_time(None), "")
2504         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2505         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2506         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2507         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2508
2509     def test_abbreviate_rate(self):
2510         self.failUnlessEqual(common.abbreviate_rate(None), "")
2511         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2512         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2513         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2514
2515     def test_abbreviate_size(self):
2516         self.failUnlessEqual(common.abbreviate_size(None), "")
2517         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2518         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2519         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2520         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2521