src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17
  18 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  19 # create a webserver that works against them
  20
  21 class FakeIntroducerClient:
  22     def get_all_connectors(self):
  23         return {}
  24     def get_all_connections_for(self, service_name):
  25         return frozenset()
  26     def get_all_peerids(self):
  27         return frozenset()
  28
  29 class FakeClient(service.MultiService):
  30     nodeid = "fake_nodeid"
  31     nickname = "fake_nickname"
  32     basedir = "fake_basedir"
  33     def get_versions(self):
  34         return {'allmydata': "fake",
  35                 'foolscap': "fake",
  36                 'twisted': "fake",
  37                 'zfec': "fake",
  38                 }
  39     introducer_furl = "None"
  40     introducer_client = FakeIntroducerClient()
  41     _all_upload_status = [upload.UploadStatus()]
  42     _all_download_status = [download.DownloadStatus()]
  43     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  44     _all_publish_statuses = [publish.PublishStatus()]
  45     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  46     convergence = "some random string"
  47
  48     def connected_to_introducer(self):
  49         return False
  50
  51     def create_node_from_uri(self, auri):
  52         u = uri.from_string(auri)
  53         if (INewDirectoryURI.providedBy(u)
  54             or IReadonlyNewDirectoryURI.providedBy(u)):
  55             return FakeDirectoryNode(self).init_from_uri(u)
  56         if IFileURI.providedBy(u):
  57             return FakeCHKFileNode(u, self)
  58         assert IMutableFileURI.providedBy(u), u
  59         return FakeMutableFileNode(self).init_from_uri(u)
  60
  61     def create_empty_dirnode(self):
  62         n = FakeDirectoryNode(self)
  63         d = n.create()
  64         d.addCallback(lambda res: n)
  65         return d
  66
  67     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  68     def create_mutable_file(self, contents=""):
  69         n = FakeMutableFileNode(self)
  70         return n.create(contents)
  71
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(self, data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83
  84     def list_all_upload_statuses(self):
  85         return self._all_upload_status
  86     def list_all_download_statuses(self):
  87         return self._all_download_status
  88     def list_all_mapupdate_statuses(self):
  89         return self._all_mapupdate_statuses
  90     def list_all_publish_statuses(self):
  91         return self._all_publish_statuses
  92     def list_all_retrieve_statuses(self):
  93         return self._all_retrieve_statuses
  94     def list_all_helper_statuses(self):
  95         return []
  96
  97 class HTTPClientHEADFactory(client.HTTPClientFactory):
  98     def __init__(self, *args, **kwargs):
  99         client.HTTPClientFactory.__init__(self, *args, **kwargs)
 100         self.deferred.addCallback(lambda res: self.response_headers)
 101
 102     def noPage(self, reason):
 103         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 104         # exception when the response to a HEAD request had no body (when in
 105         # fact they are defined to never have a body). This was fixed in
 106         # Twisted-8.0 . To work around this, we catch the
 107         # PartialDownloadError and make it disappear.
 108         if (reason.check(client.PartialDownloadError)
 109             and self.method.upper() == "HEAD"):
 110             self.page("")
 111             return
 112         return client.HTTPClientFactory.noPage(self, reason)
 113
 114
 115 class WebMixin(object):
 116     def setUp(self):
 117         self.s = FakeClient()
 118         self.s.startService()
 119         self.ws = s = webish.WebishServer("0")
 120         s.setServiceParent(self.s)
 121         self.webish_port = port = s.listener._port.getHost().port
 122         self.webish_url = "http://localhost:%d" % port
 123
 124         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 125         d = defer.DeferredList(l)
 126         def _then(res):
 127             self.public_root = res[0][1]
 128             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 129             self.public_url = "/uri/" + self.public_root.get_uri()
 130             self.private_root = res[1][1]
 131
 132             foo = res[2][1]
 133             self._foo_node = foo
 134             self._foo_uri = foo.get_uri()
 135             self._foo_readonly_uri = foo.get_readonly_uri()
 136             # NOTE: we ignore the deferred on all set_uri() calls, because we
 137             # know the fake nodes do these synchronously
 138             self.public_root.set_uri(u"foo", foo.get_uri())
 139
 140             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 141             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 142
 143             foo.set_uri(u"empty", res[3][1].get_uri())
 144             sub_uri = res[4][1].get_uri()
 145             self._sub_uri = sub_uri
 146             foo.set_uri(u"sub", sub_uri)
 147             sub = self.s.create_node_from_uri(sub_uri)
 148
 149             _ign, n, blocking_uri = self.makefile(1)
 150             foo.set_uri(u"blockingfile", blocking_uri)
 151
 152             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 153             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 154             # still think of it as an umlaut
 155             foo.set_uri(unicode_filename, self._bar_txt_uri)
 156
 157             _ign, n, baz_file = self.makefile(2)
 158             sub.set_uri(u"baz.txt", baz_file)
 159
 160             _ign, n, self._bad_file_uri = self.makefile(3)
 161             # this uri should not be downloadable
 162             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 163
 164             rodir = res[5][1]
 165             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 166             rodir.set_uri(u"nor", baz_file)
 167
 168             # public/
 169             # public/foo/
 170             # public/foo/bar.txt
 171             # public/foo/blockingfile
 172             # public/foo/empty/
 173             # public/foo/sub/
 174             # public/foo/sub/baz.txt
 175             # public/reedownlee/
 176             # public/reedownlee/nor
 177             self.NEWFILE_CONTENTS = "newfile contents\n"
 178
 179             return foo.get_metadata_for(u"bar.txt")
 180         d.addCallback(_then)
 181         def _got_metadata(metadata):
 182             self._bar_txt_metadata = metadata
 183         d.addCallback(_got_metadata)
 184         return d
 185
 186     def makefile(self, number):
 187         contents = "contents of file %s\n" % number
 188         n = create_chk_filenode(self.s, contents)
 189         return contents, n, n.get_uri()
 190
 191     def tearDown(self):
 192         return self.s.stopService()
 193
 194     def failUnlessIsBarDotTxt(self, res):
 195         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 196
 197     def failUnlessIsBarJSON(self, res):
 198         data = simplejson.loads(res)
 199         self.failUnless(isinstance(data, list))
 200         self.failUnlessEqual(data[0], "filenode")
 201         self.failUnless(isinstance(data[1], dict))
 202         self.failIf(data[1]["mutable"])
 203         self.failIf("rw_uri" in data[1]) # immutable
 204         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 205         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 206
 207     def failUnlessIsFooJSON(self, res):
 208         data = simplejson.loads(res)
 209         self.failUnless(isinstance(data, list))
 210         self.failUnlessEqual(data[0], "dirnode", res)
 211         self.failUnless(isinstance(data[1], dict))
 212         self.failUnless(data[1]["mutable"])
 213         self.failUnless("rw_uri" in data[1]) # mutable
 214         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 215         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 216
 217         kidnames = sorted(data[1]["children"])
 218         self.failUnlessEqual(kidnames,
 219                              [u"bar.txt", u"blockingfile", u"empty",
 220                               u"n\u00fc.txt", u"sub"])
 221         kids = data[1]["children"]
 222         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 223         self.failUnless("metadata" in kids[u"sub"][1])
 224         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 225         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 226         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 227         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 228         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 229         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 230                              self._bar_txt_metadata["ctime"])
 231         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 232                              self._bar_txt_uri)
 233
 234     def GET(self, urlpath, followRedirect=False):
 235         assert not isinstance(urlpath, unicode)
 236         url = self.webish_url + urlpath
 237         return client.getPage(url, method="GET", followRedirect=followRedirect)
 238
 239     def HEAD(self, urlpath):
 240         # this requires some surgery, because twisted.web.client doesn't want
 241         # to give us back the response headers.
 242         factory = HTTPClientHEADFactory(urlpath, method="HEAD")
 243         reactor.connectTCP("localhost", self.webish_port, factory)
 244         return factory.deferred
 245
 246     def PUT(self, urlpath, data):
 247         url = self.webish_url + urlpath
 248         return client.getPage(url, method="PUT", postdata=data)
 249
 250     def DELETE(self, urlpath):
 251         url = self.webish_url + urlpath
 252         return client.getPage(url, method="DELETE")
 253
 254     def POST(self, urlpath, followRedirect=False, **fields):
 255         url = self.webish_url + urlpath
 256         sepbase = "boogabooga"
 257         sep = "--" + sepbase
 258         form = []
 259         form.append(sep)
 260         form.append('Content-Disposition: form-data; name="_charset"')
 261         form.append('')
 262         form.append('UTF-8')
 263         form.append(sep)
 264         for name, value in fields.iteritems():
 265             if isinstance(value, tuple):
 266                 filename, value = value
 267                 form.append('Content-Disposition: form-data; name="%s"; '
 268                             'filename="%s"' % (name, filename.encode("utf-8")))
 269             else:
 270                 form.append('Content-Disposition: form-data; name="%s"' % name)
 271             form.append('')
 272             if isinstance(value, unicode):
 273                 value = value.encode("utf-8")
 274             else:
 275                 value = str(value)
 276             assert isinstance(value, str)
 277             form.append(value)
 278             form.append(sep)
 279         form[-1] += "--"
 280         body = "\r\n".join(form) + "\r\n"
 281         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 282                    }
 283         return client.getPage(url, method="POST", postdata=body,
 284                               headers=headers, followRedirect=followRedirect)
 285
 286     def shouldFail(self, res, expected_failure, which,
 287                    substring=None, response_substring=None):
 288         if isinstance(res, failure.Failure):
 289             res.trap(expected_failure)
 290             if substring:
 291                 self.failUnless(substring in str(res),
 292                                 "substring '%s' not in '%s'"
 293                                 % (substring, str(res)))
 294             if response_substring:
 295                 self.failUnless(response_substring in res.value.response,
 296                                 "response substring '%s' not in '%s'"
 297                                 % (response_substring, res.value.response))
 298         else:
 299             self.fail("%s was supposed to raise %s, not get '%s'" %
 300                       (which, expected_failure, res))
 301
 302     def shouldFail2(self, expected_failure, which, substring,
 303                     response_substring,
 304                     callable, *args, **kwargs):
 305         assert substring is None or isinstance(substring, str)
 306         assert response_substring is None or isinstance(response_substring, str)
 307         d = defer.maybeDeferred(callable, *args, **kwargs)
 308         def done(res):
 309             if isinstance(res, failure.Failure):
 310                 res.trap(expected_failure)
 311                 if substring:
 312                     self.failUnless(substring in str(res),
 313                                     "%s: substring '%s' not in '%s'"
 314                                     % (which, substring, str(res)))
 315                 if response_substring:
 316                     self.failUnless(response_substring in res.value.response,
 317                                     "%s: response substring '%s' not in '%s'"
 318                                     % (which,
 319                                        response_substring, res.value.response))
 320             else:
 321                 self.fail("%s was supposed to raise %s, not get '%s'" %
 322                           (which, expected_failure, res))
 323         d.addBoth(done)
 324         return d
 325
 326     def should404(self, res, which):
 327         if isinstance(res, failure.Failure):
 328             res.trap(error.Error)
 329             self.failUnlessEqual(res.value.status, "404")
 330         else:
 331             self.fail("%s was supposed to Error(404), not get '%s'" %
 332                       (which, res))
 333
 334     def shouldHTTPError(self, res, which, code=None, substring=None,
 335                         response_substring=None):
 336         if isinstance(res, failure.Failure):
 337             res.trap(error.Error)
 338             if code is not None:
 339                 self.failUnlessEqual(res.value.status, str(code))
 340             if substring:
 341                 self.failUnless(substring in str(res),
 342                                 "substring '%s' not in '%s'"
 343                                 % (substring, str(res)))
 344             if response_substring:
 345                 self.failUnless(response_substring in res.value.response,
 346                                 "response substring '%s' not in '%s'"
 347                                 % (response_substring, res.value.response))
 348         else:
 349             self.fail("%s was supposed to Error(%s), not get '%s'" %
 350                       (which, code, res))
 351
 352     def shouldHTTPError2(self, which,
 353                          code=None, substring=None, response_substring=None,
 354                          callable=None, *args, **kwargs):
 355         assert substring is None or isinstance(substring, str)
 356         assert callable
 357         d = defer.maybeDeferred(callable, *args, **kwargs)
 358         d.addBoth(self.shouldHTTPError, which,
 359                   code, substring, response_substring)
 360         return d
 361
 362
 363 class Web(WebMixin, unittest.TestCase):
 364     def test_create(self):
 365         pass
 366
 367     def test_welcome(self):
 368         d = self.GET("/")
 369         def _check(res):
 370             self.failUnless('Welcome To AllMyData' in res)
 371             self.failUnless('Tahoe' in res)
 372
 373             self.s.basedir = 'web/test_welcome'
 374             fileutil.make_dirs("web/test_welcome")
 375             fileutil.make_dirs("web/test_welcome/private")
 376             return self.GET("/")
 377         d.addCallback(_check)
 378         return d
 379
 380     def test_provisioning_math(self):
 381         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 382         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 383         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 384         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 385         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 386
 387     def test_provisioning(self):
 388         d = self.GET("/provisioning/")
 389         def _check(res):
 390             self.failUnless('Tahoe Provisioning Tool' in res)
 391             fields = {'filled': True,
 392                       "num_users": int(50e3),
 393                       "files_per_user": 1000,
 394                       "space_per_user": int(1e9),
 395                       "sharing_ratio": 1.0,
 396                       "encoding_parameters": "3-of-10-5",
 397                       "num_servers": 30,
 398                       "ownership_mode": "A",
 399                       "download_rate": 100,
 400                       "upload_rate": 10,
 401                       "delete_rate": 10,
 402                       "lease_timer": 7,
 403                       }
 404             return self.POST("/provisioning/", **fields)
 405
 406         d.addCallback(_check)
 407         def _check2(res):
 408             self.failUnless('Tahoe Provisioning Tool' in res)
 409             self.failUnless("Share space consumed: 167.01TB" in res)
 410
 411             fields = {'filled': True,
 412                       "num_users": int(50e6),
 413                       "files_per_user": 1000,
 414                       "space_per_user": int(5e9),
 415                       "sharing_ratio": 1.0,
 416                       "encoding_parameters": "25-of-100-50",
 417                       "num_servers": 30000,
 418                       "ownership_mode": "E",
 419                       "drive_failure_model": "U",
 420                       "drive_size": 1000,
 421                       "download_rate": 1000,
 422                       "upload_rate": 100,
 423                       "delete_rate": 100,
 424                       "lease_timer": 7,
 425                       }
 426             return self.POST("/provisioning/", **fields)
 427         d.addCallback(_check2)
 428         def _check3(res):
 429             self.failUnless("Share space consumed: huge!" in res)
 430             fields = {'filled': True}
 431             return self.POST("/provisioning/", **fields)
 432         d.addCallback(_check3)
 433         def _check4(res):
 434             self.failUnless("Share space consumed:" in res)
 435         d.addCallback(_check4)
 436         return d
 437
 438     def test_status(self):
 439         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 440         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 441         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 442         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 443         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 444         d = self.GET("/status", followRedirect=True)
 445         def _check(res):
 446             self.failUnless('Upload and Download Status' in res, res)
 447             self.failUnless('"down-%d"' % dl_num in res, res)
 448             self.failUnless('"up-%d"' % ul_num in res, res)
 449             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 450             self.failUnless('"publish-%d"' % pub_num in res, res)
 451             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 452         d.addCallback(_check)
 453         d.addCallback(lambda res: self.GET("/status/?t=json"))
 454         def _check_json(res):
 455             data = simplejson.loads(res)
 456             self.failUnless(isinstance(data, dict))
 457             active = data["active"]
 458             # TODO: test more. We need a way to fake an active operation
 459             # here.
 460         d.addCallback(_check_json)
 461
 462         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 463         def _check_dl(res):
 464             self.failUnless("File Download Status" in res, res)
 465         d.addCallback(_check_dl)
 466         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 467         def _check_ul(res):
 468             self.failUnless("File Upload Status" in res, res)
 469         d.addCallback(_check_ul)
 470         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 471         def _check_mapupdate(res):
 472             self.failUnless("Mutable File Servermap Update Status" in res, res)
 473         d.addCallback(_check_mapupdate)
 474         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 475         def _check_publish(res):
 476             self.failUnless("Mutable File Publish Status" in res, res)
 477         d.addCallback(_check_publish)
 478         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 479         def _check_retrieve(res):
 480             self.failUnless("Mutable File Retrieve Status" in res, res)
 481         d.addCallback(_check_retrieve)
 482
 483         return d
 484
 485     def test_status_numbers(self):
 486         drrm = status.DownloadResultsRendererMixin()
 487         self.failUnlessEqual(drrm.render_time(None, None), "")
 488         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 489         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 490         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 491         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 492         self.failUnlessEqual(drrm.render_rate(None, None), "")
 493         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 494         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 495         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 496
 497         urrm = status.UploadResultsRendererMixin()
 498         self.failUnlessEqual(urrm.render_time(None, None), "")
 499         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 500         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 501         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 502         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 503         self.failUnlessEqual(urrm.render_rate(None, None), "")
 504         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 505         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 506         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 507
 508     def test_GET_FILEURL(self):
 509         d = self.GET(self.public_url + "/foo/bar.txt")
 510         d.addCallback(self.failUnlessIsBarDotTxt)
 511         return d
 512
 513     def test_HEAD_FILEURL(self):
 514         d = self.HEAD(self.public_url + "/foo/bar.txt")
 515         def _got(headers):
 516             self.failUnlessEqual(headers["content-length"][0],
 517                                  str(len(self.BAR_CONTENTS)))
 518             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 519         d.addCallback(_got)
 520         return d
 521
 522     def test_GET_FILEURL_named(self):
 523         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 524         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 525         d = self.GET(base + "/@@name=/blah.txt")
 526         d.addCallback(self.failUnlessIsBarDotTxt)
 527         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 528         d.addCallback(self.failUnlessIsBarDotTxt)
 529         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 530         d.addCallback(self.failUnlessIsBarDotTxt)
 531         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 532         d.addCallback(self.failUnlessIsBarDotTxt)
 533         save_url = base + "?save=true&filename=blah.txt"
 534         d.addCallback(lambda res: self.GET(save_url))
 535         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 536         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 537         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 538         u_url = base + "?save=true&filename=" + u_fn_e
 539         d.addCallback(lambda res: self.GET(u_url))
 540         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 541         return d
 542
 543     def test_PUT_FILEURL_named_bad(self):
 544         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 545         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 546                              "400 Bad Request",
 547                              "/file can only be used with GET or HEAD",
 548                              self.PUT, base + "/@@name=/blah.txt", "")
 549         return d
 550
 551     def test_GET_DIRURL_named_bad(self):
 552         base = "/file/%s" % urllib.quote(self._foo_uri)
 553         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 554                              "400 Bad Request",
 555                              "is not a file-cap",
 556                              self.GET, base + "/@@name=/blah.txt")
 557         return d
 558
 559     def test_GET_slash_file_bad(self):
 560         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 561                              "404 Not Found",
 562                              "/file must be followed by a file-cap and a name",
 563                              self.GET, "/file")
 564         return d
 565
 566     def test_GET_unhandled_URI_named(self):
 567         contents, n, newuri = self.makefile(12)
 568         verifier_cap = n.get_verifier().to_string()
 569         base = "/file/%s" % urllib.quote(verifier_cap)
 570         # client.create_node_from_uri() can't handle verify-caps
 571         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 572                              "400 Bad Request",
 573                              "is not a valid file- or directory- cap",
 574                              self.GET, base)
 575         return d
 576
 577     def test_GET_unhandled_URI(self):
 578         contents, n, newuri = self.makefile(12)
 579         verifier_cap = n.get_verifier().to_string()
 580         base = "/uri/%s" % urllib.quote(verifier_cap)
 581         # client.create_node_from_uri() can't handle verify-caps
 582         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 583                              "400 Bad Request",
 584                              "is not a valid file- or directory- cap",
 585                              self.GET, base)
 586         return d
 587
 588     def test_GET_FILE_URI(self):
 589         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 590         d = self.GET(base)
 591         d.addCallback(self.failUnlessIsBarDotTxt)
 592         return d
 593
 594     def test_GET_FILE_URI_badchild(self):
 595         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 596         errmsg = "Files have no children, certainly not named 'boguschild'"
 597         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 598                              "400 Bad Request", errmsg,
 599                              self.GET, base)
 600         return d
 601
 602     def test_PUT_FILE_URI_badchild(self):
 603         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 604         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 605         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 606                              "400 Bad Request", errmsg,
 607                              self.PUT, base, "")
 608         return d
 609
 610     def test_GET_FILEURL_save(self):
 611         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 612         # TODO: look at the headers, expect a Content-Disposition: attachment
 613         # header.
 614         d.addCallback(self.failUnlessIsBarDotTxt)
 615         return d
 616
 617     def test_GET_FILEURL_missing(self):
 618         d = self.GET(self.public_url + "/foo/missing")
 619         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 620         return d
 621
 622     def test_PUT_NEWFILEURL(self):
 623         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 624         # TODO: we lose the response code, so we can't check this
 625         #self.failUnlessEqual(responsecode, 201)
 626         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 627         d.addCallback(lambda res:
 628                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 629                                                       self.NEWFILE_CONTENTS))
 630         return d
 631
 632     def test_PUT_NEWFILEURL_mutable(self):
 633         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 634                      self.NEWFILE_CONTENTS)
 635         # TODO: we lose the response code, so we can't check this
 636         #self.failUnlessEqual(responsecode, 201)
 637         def _check_uri(res):
 638             u = uri.from_string_mutable_filenode(res)
 639             self.failUnless(u.is_mutable())
 640             self.failIf(u.is_readonly())
 641             return res
 642         d.addCallback(_check_uri)
 643         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 644         d.addCallback(lambda res:
 645                       self.failUnlessMutableChildContentsAre(self._foo_node,
 646                                                              u"new.txt",
 647                                                              self.NEWFILE_CONTENTS))
 648         return d
 649
 650     def test_PUT_NEWFILEURL_mutable_toobig(self):
 651         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 652                              "413 Request Entity Too Large",
 653                              "SDMF is limited to one segment, and 10001 > 10000",
 654                              self.PUT,
 655                              self.public_url + "/foo/new.txt?mutable=true",
 656                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 657         return d
 658
 659     def test_PUT_NEWFILEURL_replace(self):
 660         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 661         # TODO: we lose the response code, so we can't check this
 662         #self.failUnlessEqual(responsecode, 200)
 663         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 664         d.addCallback(lambda res:
 665                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 666                                                       self.NEWFILE_CONTENTS))
 667         return d
 668
 669     def test_PUT_NEWFILEURL_bad_t(self):
 670         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 671                              "PUT to a file: bad t=bogus",
 672                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 673                              "contents")
 674         return d
 675
 676     def test_PUT_NEWFILEURL_no_replace(self):
 677         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 678                      self.NEWFILE_CONTENTS)
 679         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 680                   "409 Conflict",
 681                   "There was already a child by that name, and you asked me "
 682                   "to not replace it")
 683         return d
 684
 685     def test_PUT_NEWFILEURL_mkdirs(self):
 686         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 687         fn = self._foo_node
 688         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 689         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 690         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 691         d.addCallback(lambda res:
 692                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 693                                                       self.NEWFILE_CONTENTS))
 694         return d
 695
 696     def test_PUT_NEWFILEURL_blocked(self):
 697         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 698                      self.NEWFILE_CONTENTS)
 699         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 700                   "409 Conflict",
 701                   "Unable to create directory 'blockingfile': a file was in the way")
 702         return d
 703
 704     def test_DELETE_FILEURL(self):
 705         d = self.DELETE(self.public_url + "/foo/bar.txt")
 706         d.addCallback(lambda res:
 707                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 708         return d
 709
 710     def test_DELETE_FILEURL_missing(self):
 711         d = self.DELETE(self.public_url + "/foo/missing")
 712         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 713         return d
 714
 715     def test_DELETE_FILEURL_missing2(self):
 716         d = self.DELETE(self.public_url + "/missing/missing")
 717         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 718         return d
 719
 720     def test_GET_FILEURL_json(self):
 721         # twisted.web.http.parse_qs ignores any query args without an '=', so
 722         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 723         # instead. This may make it tricky to emulate the S3 interface
 724         # completely.
 725         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 726         d.addCallback(self.failUnlessIsBarJSON)
 727         return d
 728
 729     def test_GET_FILEURL_json_missing(self):
 730         d = self.GET(self.public_url + "/foo/missing?json")
 731         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 732         return d
 733
 734     def test_GET_FILEURL_uri(self):
 735         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 736         def _check(res):
 737             self.failUnlessEqual(res, self._bar_txt_uri)
 738         d.addCallback(_check)
 739         d.addCallback(lambda res:
 740                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 741         def _check2(res):
 742             # for now, for files, uris and readonly-uris are the same
 743             self.failUnlessEqual(res, self._bar_txt_uri)
 744         d.addCallback(_check2)
 745         return d
 746
 747     def test_GET_FILEURL_badtype(self):
 748         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 749                                   "bad t=bogus",
 750                                   self.GET,
 751                                   self.public_url + "/foo/bar.txt?t=bogus")
 752         return d
 753
 754     def test_GET_FILEURL_uri_missing(self):
 755         d = self.GET(self.public_url + "/foo/missing?t=uri")
 756         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 757         return d
 758
 759     def test_GET_DIRURL(self):
 760         # the addSlash means we get a redirect here
 761         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 762         ROOT = "../../.."
 763         d = self.GET(self.public_url + "/foo", followRedirect=True)
 764         def _check(res):
 765             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 766                             in res, res)
 767             # the FILE reference points to a URI, but it should end in bar.txt
 768             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 769                        (ROOT, urllib.quote(self._bar_txt_uri)))
 770             get_bar = "".join([r'<td>',
 771                                r'<a href="%s">bar.txt</a>' % bar_url,
 772                                r'</td>',
 773                                r'\s+<td>FILE</td>',
 774                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 775                                ])
 776             self.failUnless(re.search(get_bar, res), res)
 777             for line in res.split("\n"):
 778                 # find the line that contains the delete button for bar.txt
 779                 if ("form action" in line and
 780                     'value="delete"' in line and
 781                     'value="bar.txt"' in line):
 782                     # the form target should use a relative URL
 783                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 784                     self.failUnless(('action="%s"' % foo_url) in line, line)
 785                     # and the when_done= should too
 786                     #done_url = urllib.quote(???)
 787                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 788                     #                in line, line)
 789                     break
 790             else:
 791                 self.fail("unable to find delete-bar.txt line", res)
 792
 793             # the DIR reference just points to a URI
 794             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 795             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 796                        + r'\s+<td>DIR</td>')
 797             self.failUnless(re.search(get_sub, res), res)
 798         d.addCallback(_check)
 799
 800         # look at a directory which is readonly
 801         d.addCallback(lambda res:
 802                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 803         def _check2(res):
 804             self.failUnless("(readonly)" in res, res)
 805             self.failIf("Upload a file" in res, res)
 806         d.addCallback(_check2)
 807
 808         # and at a directory that contains a readonly directory
 809         d.addCallback(lambda res:
 810                       self.GET(self.public_url, followRedirect=True))
 811         def _check3(res):
 812             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 813                                       '</td>\s+<td>DIR-RO</td>', res))
 814         d.addCallback(_check3)
 815
 816         return d
 817
 818     def test_GET_DIRURL_badtype(self):
 819         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 820                                   400, "Bad Request",
 821                                   "bad t=bogus",
 822                                   self.GET,
 823                                   self.public_url + "/foo?t=bogus")
 824         return d
 825
 826     def test_GET_DIRURL_json(self):
 827         d = self.GET(self.public_url + "/foo?t=json")
 828         d.addCallback(self.failUnlessIsFooJSON)
 829         return d
 830
 831     def test_GET_DIRURL_manifest(self):
 832         d = self.GET(self.public_url + "/foo?t=manifest", followRedirect=True)
 833         def _got(manifest):
 834             self.failUnless("Refresh Capabilities" in manifest)
 835         d.addCallback(_got)
 836         return d
 837
 838     def test_GET_DIRURL_deepsize(self):
 839         d = self.GET(self.public_url + "/foo?t=deep-size", followRedirect=True)
 840         def _got(manifest):
 841             self.failUnless(re.search(r'^\d+$', manifest), manifest)
 842             self.failUnlessEqual("57", manifest)
 843         d.addCallback(_got)
 844         return d
 845
 846     def test_GET_DIRURL_deepstats(self):
 847         d = self.GET(self.public_url + "/foo?t=deep-stats", followRedirect=True)
 848         def _got(stats_json):
 849             stats = simplejson.loads(stats_json)
 850             expected = {"count-immutable-files": 3,
 851                         "count-mutable-files": 0,
 852                         "count-literal-files": 0,
 853                         "count-files": 3,
 854                         "count-directories": 3,
 855                         "size-immutable-files": 57,
 856                         "size-literal-files": 0,
 857                         #"size-directories": 1912, # varies
 858                         #"largest-directory": 1590,
 859                         "largest-directory-children": 5,
 860                         "largest-immutable-file": 19,
 861                         }
 862             for k,v in expected.iteritems():
 863                 self.failUnlessEqual(stats[k], v,
 864                                      "stats[%s] was %s, not %s" %
 865                                      (k, stats[k], v))
 866             self.failUnlessEqual(stats["size-files-histogram"],
 867                                  [ [11, 31, 3] ])
 868         d.addCallback(_got)
 869         return d
 870
 871     def test_GET_DIRURL_uri(self):
 872         d = self.GET(self.public_url + "/foo?t=uri")
 873         def _check(res):
 874             self.failUnlessEqual(res, self._foo_uri)
 875         d.addCallback(_check)
 876         return d
 877
 878     def test_GET_DIRURL_readonly_uri(self):
 879         d = self.GET(self.public_url + "/foo?t=readonly-uri")
 880         def _check(res):
 881             self.failUnlessEqual(res, self._foo_readonly_uri)
 882         d.addCallback(_check)
 883         return d
 884
 885     def test_PUT_NEWDIRURL(self):
 886         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
 887         d.addCallback(lambda res:
 888                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
 889         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
 890         d.addCallback(self.failUnlessNodeKeysAre, [])
 891         return d
 892
 893     def test_PUT_NEWDIRURL_exists(self):
 894         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
 895         d.addCallback(lambda res:
 896                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 897         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 898         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 899         return d
 900
 901     def test_PUT_NEWDIRURL_blocked(self):
 902         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
 903                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
 904                              self.PUT,
 905                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
 906         d.addCallback(lambda res:
 907                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 908         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 909         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 910         return d
 911
 912     def test_PUT_NEWDIRURL_mkdir_p(self):
 913         d = defer.succeed(None)
 914         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
 915         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
 916         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
 917         def mkdir_p(mkpnode):
 918             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
 919             d = self.POST(url)
 920             def made_subsub(ssuri):
 921                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
 922                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
 923                 d = self.POST(url)
 924                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
 925                 return d
 926             d.addCallback(made_subsub)
 927             return d
 928         d.addCallback(mkdir_p)
 929         return d
 930
 931     def test_PUT_NEWDIRURL_mkdirs(self):
 932         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
 933         d.addCallback(lambda res:
 934                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
 935         d.addCallback(lambda res:
 936                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
 937         d.addCallback(lambda res:
 938                       self._foo_node.get_child_at_path(u"subdir/newdir"))
 939         d.addCallback(self.failUnlessNodeKeysAre, [])
 940         return d
 941
 942     def test_DELETE_DIRURL(self):
 943         d = self.DELETE(self.public_url + "/foo")
 944         d.addCallback(lambda res:
 945                       self.failIfNodeHasChild(self.public_root, u"foo"))
 946         return d
 947
 948     def test_DELETE_DIRURL_missing(self):
 949         d = self.DELETE(self.public_url + "/foo/missing")
 950         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
 951         d.addCallback(lambda res:
 952                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
 953         return d
 954
 955     def test_DELETE_DIRURL_missing2(self):
 956         d = self.DELETE(self.public_url + "/missing")
 957         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
 958         return d
 959
 960     def dump_root(self):
 961         print "NODEWALK"
 962         w = webish.DirnodeWalkerMixin()
 963         def visitor(childpath, childnode, metadata):
 964             print childpath
 965         d = w.walk(self.public_root, visitor)
 966         return d
 967
 968     def failUnlessNodeKeysAre(self, node, expected_keys):
 969         for k in expected_keys:
 970             assert isinstance(k, unicode)
 971         d = node.list()
 972         def _check(children):
 973             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
 974         d.addCallback(_check)
 975         return d
 976     def failUnlessNodeHasChild(self, node, name):
 977         assert isinstance(name, unicode)
 978         d = node.list()
 979         def _check(children):
 980             self.failUnless(name in children)
 981         d.addCallback(_check)
 982         return d
 983     def failIfNodeHasChild(self, node, name):
 984         assert isinstance(name, unicode)
 985         d = node.list()
 986         def _check(children):
 987             self.failIf(name in children)
 988         d.addCallback(_check)
 989         return d
 990
 991     def failUnlessChildContentsAre(self, node, name, expected_contents):
 992         assert isinstance(name, unicode)
 993         d = node.get_child_at_path(name)
 994         d.addCallback(lambda node: node.download_to_data())
 995         def _check(contents):
 996             self.failUnlessEqual(contents, expected_contents)
 997         d.addCallback(_check)
 998         return d
 999
1000     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1001         assert isinstance(name, unicode)
1002         d = node.get_child_at_path(name)
1003         d.addCallback(lambda node: node.download_best_version())
1004         def _check(contents):
1005             self.failUnlessEqual(contents, expected_contents)
1006         d.addCallback(_check)
1007         return d
1008
1009     def failUnlessChildURIIs(self, node, name, expected_uri):
1010         assert isinstance(name, unicode)
1011         d = node.get_child_at_path(name)
1012         def _check(child):
1013             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1014         d.addCallback(_check)
1015         return d
1016
1017     def failUnlessURIMatchesChild(self, got_uri, node, name):
1018         assert isinstance(name, unicode)
1019         d = node.get_child_at_path(name)
1020         def _check(child):
1021             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1022         d.addCallback(_check)
1023         return d
1024
1025     def failUnlessCHKURIHasContents(self, got_uri, contents):
1026         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1027
1028     def test_POST_upload(self):
1029         d = self.POST(self.public_url + "/foo", t="upload",
1030                       file=("new.txt", self.NEWFILE_CONTENTS))
1031         fn = self._foo_node
1032         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1033         d.addCallback(lambda res:
1034                       self.failUnlessChildContentsAre(fn, u"new.txt",
1035                                                       self.NEWFILE_CONTENTS))
1036         return d
1037
1038     def test_POST_upload_unicode(self):
1039         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1040         d = self.POST(self.public_url + "/foo", t="upload",
1041                       file=(filename, self.NEWFILE_CONTENTS))
1042         fn = self._foo_node
1043         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1044         d.addCallback(lambda res:
1045                       self.failUnlessChildContentsAre(fn, filename,
1046                                                       self.NEWFILE_CONTENTS))
1047         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1048         d.addCallback(lambda res: self.GET(target_url))
1049         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1050                                                             self.NEWFILE_CONTENTS,
1051                                                             contents))
1052         return d
1053
1054     def test_POST_upload_unicode_named(self):
1055         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1056         d = self.POST(self.public_url + "/foo", t="upload",
1057                       name=filename,
1058                       file=("overridden", self.NEWFILE_CONTENTS))
1059         fn = self._foo_node
1060         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1061         d.addCallback(lambda res:
1062                       self.failUnlessChildContentsAre(fn, filename,
1063                                                       self.NEWFILE_CONTENTS))
1064         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1065         d.addCallback(lambda res: self.GET(target_url))
1066         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1067                                                             self.NEWFILE_CONTENTS,
1068                                                             contents))
1069         return d
1070
1071     def test_POST_upload_no_link(self):
1072         d = self.POST("/uri", t="upload",
1073                       file=("new.txt", self.NEWFILE_CONTENTS))
1074         def _check_upload_results(page):
1075             # this should be a page which describes the results of the upload
1076             # that just finished.
1077             self.failUnless("Upload Results:" in page)
1078             self.failUnless("URI:" in page)
1079             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1080             mo = uri_re.search(page)
1081             self.failUnless(mo, page)
1082             new_uri = mo.group(1)
1083             return new_uri
1084         d.addCallback(_check_upload_results)
1085         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1086         return d
1087
1088     def test_POST_upload_no_link_whendone(self):
1089         d = self.POST("/uri", t="upload", when_done="/",
1090                       file=("new.txt", self.NEWFILE_CONTENTS))
1091         d.addBoth(self.shouldRedirect, "/")
1092         return d
1093
1094     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1095         d = defer.maybeDeferred(callable, *args, **kwargs)
1096         def done(res):
1097             if isinstance(res, failure.Failure):
1098                 res.trap(error.PageRedirect)
1099                 statuscode = res.value.status
1100                 target = res.value.location
1101                 return checker(statuscode, target)
1102             self.fail("%s: callable was supposed to redirect, not return '%s'"
1103                       % (which, res))
1104         d.addBoth(done)
1105         return d
1106
1107     def test_POST_upload_no_link_whendone_results(self):
1108         def check(statuscode, target):
1109             self.failUnlessEqual(statuscode, str(http.FOUND))
1110             self.failUnless(target.startswith(self.webish_url), target)
1111             return client.getPage(target, method="GET")
1112         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1113                                  check,
1114                                  self.POST, "/uri", t="upload",
1115                                  when_done="/uri/%(uri)s",
1116                                  file=("new.txt", self.NEWFILE_CONTENTS))
1117         d.addCallback(lambda res:
1118                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1119         return d
1120
1121     def test_POST_upload_no_link_mutable(self):
1122         d = self.POST("/uri", t="upload", mutable="true",
1123                       file=("new.txt", self.NEWFILE_CONTENTS))
1124         def _check(new_uri):
1125             new_uri = new_uri.strip()
1126             self.new_uri = new_uri
1127             u = IURI(new_uri)
1128             self.failUnless(IMutableFileURI.providedBy(u))
1129             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1130             n = self.s.create_node_from_uri(new_uri)
1131             return n.download_best_version()
1132         d.addCallback(_check)
1133         def _check2(data):
1134             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1135             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1136         d.addCallback(_check2)
1137         def _check3(data):
1138             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1139             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1140         d.addCallback(_check3)
1141         def _check4(data):
1142             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1143         d.addCallback(_check4)
1144         return d
1145
1146     def test_POST_upload_no_link_mutable_toobig(self):
1147         d = self.shouldFail2(error.Error,
1148                              "test_POST_upload_no_link_mutable_toobig",
1149                              "413 Request Entity Too Large",
1150                              "SDMF is limited to one segment, and 10001 > 10000",
1151                              self.POST,
1152                              "/uri", t="upload", mutable="true",
1153                              file=("new.txt",
1154                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1155         return d
1156
1157     def test_POST_upload_mutable(self):
1158         # this creates a mutable file
1159         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1160                       file=("new.txt", self.NEWFILE_CONTENTS))
1161         fn = self._foo_node
1162         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1163         d.addCallback(lambda res:
1164                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1165                                                              self.NEWFILE_CONTENTS))
1166         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1167         def _got(newnode):
1168             self.failUnless(IMutableFileNode.providedBy(newnode))
1169             self.failUnless(newnode.is_mutable())
1170             self.failIf(newnode.is_readonly())
1171             self._mutable_node = newnode
1172             self._mutable_uri = newnode.get_uri()
1173         d.addCallback(_got)
1174
1175         # now upload it again and make sure that the URI doesn't change
1176         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1177         d.addCallback(lambda res:
1178                       self.POST(self.public_url + "/foo", t="upload",
1179                                 mutable="true",
1180                                 file=("new.txt", NEWER_CONTENTS)))
1181         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1182         d.addCallback(lambda res:
1183                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1184                                                              NEWER_CONTENTS))
1185         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1186         def _got2(newnode):
1187             self.failUnless(IMutableFileNode.providedBy(newnode))
1188             self.failUnless(newnode.is_mutable())
1189             self.failIf(newnode.is_readonly())
1190             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1191         d.addCallback(_got2)
1192
1193         # upload a second time, using PUT instead of POST
1194         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1195         d.addCallback(lambda res:
1196                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1197         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1198         d.addCallback(lambda res:
1199                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1200                                                              NEW2_CONTENTS))
1201
1202         # finally list the directory, since mutable files are displayed
1203         # slightly differently
1204
1205         d.addCallback(lambda res:
1206                       self.GET(self.public_url + "/foo/",
1207                                followRedirect=True))
1208         def _check_page(res):
1209             # TODO: assert more about the contents
1210             self.failUnless("SSK" in res)
1211             return res
1212         d.addCallback(_check_page)
1213
1214         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1215         def _got3(newnode):
1216             self.failUnless(IMutableFileNode.providedBy(newnode))
1217             self.failUnless(newnode.is_mutable())
1218             self.failIf(newnode.is_readonly())
1219             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1220         d.addCallback(_got3)
1221
1222         # look at the JSON form of the enclosing directory
1223         d.addCallback(lambda res:
1224                       self.GET(self.public_url + "/foo/?t=json",
1225                                followRedirect=True))
1226         def _check_page_json(res):
1227             parsed = simplejson.loads(res)
1228             self.failUnlessEqual(parsed[0], "dirnode")
1229             children = parsed[1]["children"]
1230             self.failUnless("new.txt" in children)
1231             new_json = children["new.txt"]
1232             self.failUnlessEqual(new_json[0], "filenode")
1233             self.failUnless(new_json[1]["mutable"])
1234             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1235             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1236             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1237         d.addCallback(_check_page_json)
1238
1239         # and the JSON form of the file
1240         d.addCallback(lambda res:
1241                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1242         def _check_file_json(res):
1243             parsed = simplejson.loads(res)
1244             self.failUnlessEqual(parsed[0], "filenode")
1245             self.failUnless(parsed[1]["mutable"])
1246             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1247             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1248             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1249         d.addCallback(_check_file_json)
1250
1251         # and look at t=uri and t=readonly-uri
1252         d.addCallback(lambda res:
1253                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1254         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1255         d.addCallback(lambda res:
1256                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1257         def _check_ro_uri(res):
1258             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1259             self.failUnlessEqual(res, ro_uri)
1260         d.addCallback(_check_ro_uri)
1261
1262         # make sure we can get to it from /uri/URI
1263         d.addCallback(lambda res:
1264                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1265         d.addCallback(lambda res:
1266                       self.failUnlessEqual(res, NEW2_CONTENTS))
1267
1268         # and that HEAD computes the size correctly
1269         d.addCallback(lambda res:
1270                       self.HEAD(self.public_url + "/foo/new.txt"))
1271         def _got_headers(headers):
1272             self.failUnlessEqual(headers["content-length"][0],
1273                                  str(len(NEW2_CONTENTS)))
1274             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1275         d.addCallback(_got_headers)
1276
1277         # make sure that size errors are displayed correctly for overwrite
1278         d.addCallback(lambda res:
1279                       self.shouldFail2(error.Error,
1280                                        "test_POST_upload_mutable-toobig",
1281                                        "413 Request Entity Too Large",
1282                                        "SDMF is limited to one segment, and 10001 > 10000",
1283                                        self.POST,
1284                                        self.public_url + "/foo", t="upload",
1285                                        mutable="true",
1286                                        file=("new.txt",
1287                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1288                                        ))
1289
1290         d.addErrback(self.dump_error)
1291         return d
1292
1293     def test_POST_upload_mutable_toobig(self):
1294         d = self.shouldFail2(error.Error,
1295                              "test_POST_upload_no_link_mutable_toobig",
1296                              "413 Request Entity Too Large",
1297                              "SDMF is limited to one segment, and 10001 > 10000",
1298                              self.POST,
1299                              self.public_url + "/foo",
1300                              t="upload", mutable="true",
1301                              file=("new.txt",
1302                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1303         return d
1304
1305     def dump_error(self, f):
1306         # if the web server returns an error code (like 400 Bad Request),
1307         # web.client.getPage puts the HTTP response body into the .response
1308         # attribute of the exception object that it gives back. It does not
1309         # appear in the Failure's repr(), so the ERROR that trial displays
1310         # will be rather terse and unhelpful. addErrback this method to the
1311         # end of your chain to get more information out of these errors.
1312         if f.check(error.Error):
1313             print "web.error.Error:"
1314             print f
1315             print f.value.response
1316         return f
1317
1318     def test_POST_upload_replace(self):
1319         d = self.POST(self.public_url + "/foo", t="upload",
1320                       file=("bar.txt", self.NEWFILE_CONTENTS))
1321         fn = self._foo_node
1322         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1323         d.addCallback(lambda res:
1324                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1325                                                       self.NEWFILE_CONTENTS))
1326         return d
1327
1328     def test_POST_upload_no_replace_ok(self):
1329         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1330                       file=("new.txt", self.NEWFILE_CONTENTS))
1331         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1332         d.addCallback(lambda res: self.failUnlessEqual(res,
1333                                                        self.NEWFILE_CONTENTS))
1334         return d
1335
1336     def test_POST_upload_no_replace_queryarg(self):
1337         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1338                       file=("bar.txt", self.NEWFILE_CONTENTS))
1339         d.addBoth(self.shouldFail, error.Error,
1340                   "POST_upload_no_replace_queryarg",
1341                   "409 Conflict",
1342                   "There was already a child by that name, and you asked me "
1343                   "to not replace it")
1344         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1345         d.addCallback(self.failUnlessIsBarDotTxt)
1346         return d
1347
1348     def test_POST_upload_no_replace_field(self):
1349         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1350                       file=("bar.txt", self.NEWFILE_CONTENTS))
1351         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1352                   "409 Conflict",
1353                   "There was already a child by that name, and you asked me "
1354                   "to not replace it")
1355         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1356         d.addCallback(self.failUnlessIsBarDotTxt)
1357         return d
1358
1359     def test_POST_upload_whendone(self):
1360         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1361                       file=("new.txt", self.NEWFILE_CONTENTS))
1362         d.addBoth(self.shouldRedirect, "/THERE")
1363         fn = self._foo_node
1364         d.addCallback(lambda res:
1365                       self.failUnlessChildContentsAre(fn, u"new.txt",
1366                                                       self.NEWFILE_CONTENTS))
1367         return d
1368
1369     def test_POST_upload_named(self):
1370         fn = self._foo_node
1371         d = self.POST(self.public_url + "/foo", t="upload",
1372                       name="new.txt", file=self.NEWFILE_CONTENTS)
1373         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1374         d.addCallback(lambda res:
1375                       self.failUnlessChildContentsAre(fn, u"new.txt",
1376                                                       self.NEWFILE_CONTENTS))
1377         return d
1378
1379     def test_POST_upload_named_badfilename(self):
1380         d = self.POST(self.public_url + "/foo", t="upload",
1381                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1382         d.addBoth(self.shouldFail, error.Error,
1383                   "test_POST_upload_named_badfilename",
1384                   "400 Bad Request",
1385                   "name= may not contain a slash",
1386                   )
1387         # make sure that nothing was added
1388         d.addCallback(lambda res:
1389                       self.failUnlessNodeKeysAre(self._foo_node,
1390                                                  [u"bar.txt", u"blockingfile",
1391                                                   u"empty", u"n\u00fc.txt",
1392                                                   u"sub"]))
1393         return d
1394
1395     def test_POST_FILEURL_check(self):
1396         bar_url = self.public_url + "/foo/bar.txt"
1397         d = self.POST(bar_url, t="check")
1398         def _check(res):
1399             self.failUnless("Healthy!" in res)
1400         d.addCallback(_check)
1401         redir_url = "http://allmydata.org/TARGET"
1402         def _check2(statuscode, target):
1403             self.failUnlessEqual(statuscode, str(http.FOUND))
1404             self.failUnlessEqual(target, redir_url)
1405         d.addCallback(lambda res:
1406                       self.shouldRedirect2("test_POST_FILEURL_check",
1407                                            _check2,
1408                                            self.POST, bar_url,
1409                                            t="check",
1410                                            when_done=redir_url))
1411         d.addCallback(lambda res:
1412                       self.POST(bar_url, t="check", return_to=redir_url))
1413         def _check3(res):
1414             self.failUnless("Healthy!" in res)
1415             self.failUnless("Return to parent directory" in res)
1416             self.failUnless(redir_url in res)
1417         d.addCallback(_check3)
1418         return d
1419
1420     def test_POST_FILEURL_check_and_repair(self):
1421         bar_url = self.public_url + "/foo/bar.txt"
1422         d = self.POST(bar_url, t="check", repair="true")
1423         def _check(res):
1424             self.failUnless("Healthy!" in res)
1425         d.addCallback(_check)
1426         redir_url = "http://allmydata.org/TARGET"
1427         def _check2(statuscode, target):
1428             self.failUnlessEqual(statuscode, str(http.FOUND))
1429             self.failUnlessEqual(target, redir_url)
1430         d.addCallback(lambda res:
1431                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1432                                            _check2,
1433                                            self.POST, bar_url,
1434                                            t="check", repair="true",
1435                                            when_done=redir_url))
1436         d.addCallback(lambda res:
1437                       self.POST(bar_url, t="check", return_to=redir_url))
1438         def _check3(res):
1439             self.failUnless("Healthy!" in res)
1440             self.failUnless("Return to parent directory" in res)
1441             self.failUnless(redir_url in res)
1442         d.addCallback(_check3)
1443         return d
1444
1445     def test_POST_DIRURL_check(self):
1446         foo_url = self.public_url + "/foo/"
1447         d = self.POST(foo_url, t="check")
1448         def _check(res):
1449             self.failUnless("Healthy!" in res)
1450         d.addCallback(_check)
1451         redir_url = "http://allmydata.org/TARGET"
1452         def _check2(statuscode, target):
1453             self.failUnlessEqual(statuscode, str(http.FOUND))
1454             self.failUnlessEqual(target, redir_url)
1455         d.addCallback(lambda res:
1456                       self.shouldRedirect2("test_POST_DIRURL_check",
1457                                            _check2,
1458                                            self.POST, foo_url,
1459                                            t="check",
1460                                            when_done=redir_url))
1461         d.addCallback(lambda res:
1462                       self.POST(foo_url, t="check", return_to=redir_url))
1463         def _check3(res):
1464             self.failUnless("Healthy!" in res)
1465             self.failUnless("Return to parent directory" in res)
1466             self.failUnless(redir_url in res)
1467         d.addCallback(_check3)
1468         return d
1469
1470     def test_POST_DIRURL_check_and_repair(self):
1471         foo_url = self.public_url + "/foo/"
1472         d = self.POST(foo_url, t="check", repair="true")
1473         def _check(res):
1474             self.failUnless("Healthy!" in res)
1475         d.addCallback(_check)
1476         redir_url = "http://allmydata.org/TARGET"
1477         def _check2(statuscode, target):
1478             self.failUnlessEqual(statuscode, str(http.FOUND))
1479             self.failUnlessEqual(target, redir_url)
1480         d.addCallback(lambda res:
1481                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1482                                            _check2,
1483                                            self.POST, foo_url,
1484                                            t="check", repair="true",
1485                                            when_done=redir_url))
1486         d.addCallback(lambda res:
1487                       self.POST(foo_url, t="check", return_to=redir_url))
1488         def _check3(res):
1489             self.failUnless("Healthy!" in res)
1490             self.failUnless("Return to parent directory" in res)
1491             self.failUnless(redir_url in res)
1492         d.addCallback(_check3)
1493         return d
1494
1495     def test_POST_DIRURL_deepcheck(self):
1496         d = self.POST(self.public_url, t="deep-check")
1497         def _check(res):
1498             self.failUnless("Objects Checked: <span>8</span>" in res)
1499             self.failUnless("Objects Healthy: <span>8</span>" in res)
1500         d.addCallback(_check)
1501         redir_url = "http://allmydata.org/TARGET"
1502         def _check2(statuscode, target):
1503             self.failUnlessEqual(statuscode, str(http.FOUND))
1504             self.failUnlessEqual(target, redir_url)
1505         d.addCallback(lambda res:
1506                       self.shouldRedirect2("test_POST_DIRURL_check",
1507                                            _check2,
1508                                            self.POST, self.public_url,
1509                                            t="deep-check",
1510                                            when_done=redir_url))
1511         d.addCallback(lambda res:
1512                       self.POST(self.public_url, t="deep-check",
1513                                 return_to=redir_url))
1514         def _check3(res):
1515             self.failUnless("Return to parent directory" in res)
1516             self.failUnless(redir_url in res)
1517         d.addCallback(_check3)
1518         return d
1519
1520     def test_POST_DIRURL_deepcheck_and_repair(self):
1521         d = self.POST(self.public_url, t="deep-check", repair="true")
1522         def _check(res):
1523             self.failUnless("Objects Checked: <span>8</span>" in res)
1524
1525             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1526             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1527             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1528
1529             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1530             self.failUnless("Repairs Successful: <span>0</span>" in res)
1531             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1532
1533             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1534             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1535             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1536         d.addCallback(_check)
1537         redir_url = "http://allmydata.org/TARGET"
1538         def _check2(statuscode, target):
1539             self.failUnlessEqual(statuscode, str(http.FOUND))
1540             self.failUnlessEqual(target, redir_url)
1541         d.addCallback(lambda res:
1542                       self.shouldRedirect2("test_POST_DIRURL_check",
1543                                            _check2,
1544                                            self.POST, self.public_url,
1545                                            t="deep-check",
1546                                            when_done=redir_url))
1547         d.addCallback(lambda res:
1548                       self.POST(self.public_url, t="deep-check",
1549                                 return_to=redir_url))
1550         def _check3(res):
1551             self.failUnless("Return to parent directory" in res)
1552             self.failUnless(redir_url in res)
1553         d.addCallback(_check3)
1554         return d
1555
1556     def test_POST_FILEURL_bad_t(self):
1557         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1558                              "POST to file: bad t=bogus",
1559                              self.POST, self.public_url + "/foo/bar.txt",
1560                              t="bogus")
1561         return d
1562
1563     def test_POST_mkdir(self): # return value?
1564         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1565         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1566         d.addCallback(self.failUnlessNodeKeysAre, [])
1567         return d
1568
1569     def test_POST_mkdir_2(self):
1570         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1571         d.addCallback(lambda res:
1572                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1573         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1574         d.addCallback(self.failUnlessNodeKeysAre, [])
1575         return d
1576
1577     def test_POST_mkdirs_2(self):
1578         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1579         d.addCallback(lambda res:
1580                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1581         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1582         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1583         d.addCallback(self.failUnlessNodeKeysAre, [])
1584         return d
1585
1586     def test_POST_mkdir_no_parentdir_noredirect(self):
1587         d = self.POST("/uri?t=mkdir")
1588         def _after_mkdir(res):
1589             uri.NewDirectoryURI.init_from_string(res)
1590         d.addCallback(_after_mkdir)
1591         return d
1592
1593     def test_POST_mkdir_no_parentdir_redirect(self):
1594         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1595         d.addBoth(self.shouldRedirect, None, statuscode='303')
1596         def _check_target(target):
1597             target = urllib.unquote(target)
1598             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1599         d.addCallback(_check_target)
1600         return d
1601
1602     def test_POST_noparent_bad(self):
1603         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1604                                   "/uri accepts only PUT, PUT?t=mkdir, "
1605                                   "POST?t=upload, and POST?t=mkdir",
1606                                   self.POST, "/uri?t=bogus")
1607         return d
1608
1609     def test_welcome_page_mkdir_button(self):
1610         # Fetch the welcome page.
1611         d = self.GET("/")
1612         def _after_get_welcome_page(res):
1613             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1614             mo = MKDIR_BUTTON_RE.search(res)
1615             formaction = mo.group(1)
1616             formt = mo.group(2)
1617             formaname = mo.group(3)
1618             formavalue = mo.group(4)
1619             return (formaction, formt, formaname, formavalue)
1620         d.addCallback(_after_get_welcome_page)
1621         def _after_parse_form(res):
1622             (formaction, formt, formaname, formavalue) = res
1623             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1624         d.addCallback(_after_parse_form)
1625         d.addBoth(self.shouldRedirect, None, statuscode='303')
1626         return d
1627
1628     def test_POST_mkdir_replace(self): # return value?
1629         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1630         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1631         d.addCallback(self.failUnlessNodeKeysAre, [])
1632         return d
1633
1634     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1635         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1636         d.addBoth(self.shouldFail, error.Error,
1637                   "POST_mkdir_no_replace_queryarg",
1638                   "409 Conflict",
1639                   "There was already a child by that name, and you asked me "
1640                   "to not replace it")
1641         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1642         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1643         return d
1644
1645     def test_POST_mkdir_no_replace_field(self): # return value?
1646         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1647                       replace="false")
1648         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1649                   "409 Conflict",
1650                   "There was already a child by that name, and you asked me "
1651                   "to not replace it")
1652         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1653         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1654         return d
1655
1656     def test_POST_mkdir_whendone_field(self):
1657         d = self.POST(self.public_url + "/foo",
1658                       t="mkdir", name="newdir", when_done="/THERE")
1659         d.addBoth(self.shouldRedirect, "/THERE")
1660         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1661         d.addCallback(self.failUnlessNodeKeysAre, [])
1662         return d
1663
1664     def test_POST_mkdir_whendone_queryarg(self):
1665         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1666                       t="mkdir", name="newdir")
1667         d.addBoth(self.shouldRedirect, "/THERE")
1668         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1669         d.addCallback(self.failUnlessNodeKeysAre, [])
1670         return d
1671
1672     def test_POST_bad_t(self):
1673         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1674                              "POST to a directory with bad t=BOGUS",
1675                              self.POST, self.public_url + "/foo", t="BOGUS")
1676         return d
1677
1678     def test_POST_set_children(self):
1679         contents9, n9, newuri9 = self.makefile(9)
1680         contents10, n10, newuri10 = self.makefile(10)
1681         contents11, n11, newuri11 = self.makefile(11)
1682
1683         reqbody = """{
1684                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1685                                                 "size": 0,
1686                                                 "metadata": {
1687                                                   "ctime": 1002777696.7564139,
1688                                                   "mtime": 1002777696.7564139
1689                                                  }
1690                                                } ],
1691                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1692                                                 "size": 1,
1693                                                 "metadata": {
1694                                                   "ctime": 1002777696.7564139,
1695                                                   "mtime": 1002777696.7564139
1696                                                  }
1697                                                } ],
1698                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1699                                                 "size": 2,
1700                                                 "metadata": {
1701                                                   "ctime": 1002777696.7564139,
1702                                                   "mtime": 1002777696.7564139
1703                                                  }
1704                                                } ]
1705                     }""" % (newuri9, newuri10, newuri11)
1706
1707         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1708
1709         d = client.getPage(url, method="POST", postdata=reqbody)
1710         def _then(res):
1711             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1712             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1713             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1714
1715         d.addCallback(_then)
1716         d.addErrback(self.dump_error)
1717         return d
1718
1719     def test_POST_put_uri(self):
1720         contents, n, newuri = self.makefile(8)
1721         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1722         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1723         d.addCallback(lambda res:
1724                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1725                                                       contents))
1726         return d
1727
1728     def test_POST_put_uri_replace(self):
1729         contents, n, newuri = self.makefile(8)
1730         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1731         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1732         d.addCallback(lambda res:
1733                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1734                                                       contents))
1735         return d
1736
1737     def test_POST_put_uri_no_replace_queryarg(self):
1738         contents, n, newuri = self.makefile(8)
1739         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1740                       name="bar.txt", uri=newuri)
1741         d.addBoth(self.shouldFail, error.Error,
1742                   "POST_put_uri_no_replace_queryarg",
1743                   "409 Conflict",
1744                   "There was already a child by that name, and you asked me "
1745                   "to not replace it")
1746         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1747         d.addCallback(self.failUnlessIsBarDotTxt)
1748         return d
1749
1750     def test_POST_put_uri_no_replace_field(self):
1751         contents, n, newuri = self.makefile(8)
1752         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1753                       name="bar.txt", uri=newuri)
1754         d.addBoth(self.shouldFail, error.Error,
1755                   "POST_put_uri_no_replace_field",
1756                   "409 Conflict",
1757                   "There was already a child by that name, and you asked me "
1758                   "to not replace it")
1759         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1760         d.addCallback(self.failUnlessIsBarDotTxt)
1761         return d
1762
1763     def test_POST_delete(self):
1764         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1765         d.addCallback(lambda res: self._foo_node.list())
1766         def _check(children):
1767             self.failIf(u"bar.txt" in children)
1768         d.addCallback(_check)
1769         return d
1770
1771     def test_POST_rename_file(self):
1772         d = self.POST(self.public_url + "/foo", t="rename",
1773                       from_name="bar.txt", to_name='wibble.txt')
1774         d.addCallback(lambda res:
1775                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1776         d.addCallback(lambda res:
1777                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1778         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1779         d.addCallback(self.failUnlessIsBarDotTxt)
1780         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1781         d.addCallback(self.failUnlessIsBarJSON)
1782         return d
1783
1784     def test_POST_rename_file_replace(self):
1785         # rename a file and replace a directory with it
1786         d = self.POST(self.public_url + "/foo", t="rename",
1787                       from_name="bar.txt", to_name='empty')
1788         d.addCallback(lambda res:
1789                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1790         d.addCallback(lambda res:
1791                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
1792         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
1793         d.addCallback(self.failUnlessIsBarDotTxt)
1794         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1795         d.addCallback(self.failUnlessIsBarJSON)
1796         return d
1797
1798     def test_POST_rename_file_no_replace_queryarg(self):
1799         # rename a file and replace a directory with it
1800         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
1801                       from_name="bar.txt", to_name='empty')
1802         d.addBoth(self.shouldFail, error.Error,
1803                   "POST_rename_file_no_replace_queryarg",
1804                   "409 Conflict",
1805                   "There was already a child by that name, and you asked me "
1806                   "to not replace it")
1807         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1808         d.addCallback(self.failUnlessIsEmptyJSON)
1809         return d
1810
1811     def test_POST_rename_file_no_replace_field(self):
1812         # rename a file and replace a directory with it
1813         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
1814                       from_name="bar.txt", to_name='empty')
1815         d.addBoth(self.shouldFail, error.Error,
1816                   "POST_rename_file_no_replace_field",
1817                   "409 Conflict",
1818                   "There was already a child by that name, and you asked me "
1819                   "to not replace it")
1820         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1821         d.addCallback(self.failUnlessIsEmptyJSON)
1822         return d
1823
1824     def failUnlessIsEmptyJSON(self, res):
1825         data = simplejson.loads(res)
1826         self.failUnlessEqual(data[0], "dirnode", data)
1827         self.failUnlessEqual(len(data[1]["children"]), 0)
1828
1829     def test_POST_rename_file_slash_fail(self):
1830         d = self.POST(self.public_url + "/foo", t="rename",
1831                       from_name="bar.txt", to_name='kirk/spock.txt')
1832         d.addBoth(self.shouldFail, error.Error,
1833                   "test_POST_rename_file_slash_fail",
1834                   "400 Bad Request",
1835                   "to_name= may not contain a slash",
1836                   )
1837         d.addCallback(lambda res:
1838                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1839         return d
1840
1841     def test_POST_rename_dir(self):
1842         d = self.POST(self.public_url, t="rename",
1843                       from_name="foo", to_name='plunk')
1844         d.addCallback(lambda res:
1845                       self.failIfNodeHasChild(self.public_root, u"foo"))
1846         d.addCallback(lambda res:
1847                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
1848         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
1849         d.addCallback(self.failUnlessIsFooJSON)
1850         return d
1851
1852     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
1853         """ If target is not None then the redirection has to go to target.  If
1854         statuscode is not None then the redirection has to be accomplished with
1855         that HTTP status code."""
1856         if not isinstance(res, failure.Failure):
1857             to_where = (target is None) and "somewhere" or ("to " + target)
1858             self.fail("%s: we were expecting to get redirected %s, not get an"
1859                       " actual page: %s" % (which, to_where, res))
1860         res.trap(error.PageRedirect)
1861         if statuscode is not None:
1862             self.failUnlessEqual(res.value.status, statuscode,
1863                                  "%s: not a redirect" % which)
1864         if target is not None:
1865             # the PageRedirect does not seem to capture the uri= query arg
1866             # properly, so we can't check for it.
1867             realtarget = self.webish_url + target
1868             self.failUnlessEqual(res.value.location, realtarget,
1869                                  "%s: wrong target" % which)
1870         return res.value.location
1871
1872     def test_GET_URI_form(self):
1873         base = "/uri?uri=%s" % self._bar_txt_uri
1874         # this is supposed to give us a redirect to /uri/$URI, plus arguments
1875         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1876         d = self.GET(base)
1877         d.addBoth(self.shouldRedirect, targetbase)
1878         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
1879         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
1880         d.addCallback(lambda res: self.GET(base+"&t=json"))
1881         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
1882         d.addCallback(self.log, "about to get file by uri")
1883         d.addCallback(lambda res: self.GET(base, followRedirect=True))
1884         d.addCallback(self.failUnlessIsBarDotTxt)
1885         d.addCallback(self.log, "got file by uri, about to get dir by uri")
1886         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
1887                                            followRedirect=True))
1888         d.addCallback(self.failUnlessIsFooJSON)
1889         d.addCallback(self.log, "got dir by uri")
1890
1891         return d
1892
1893     def test_GET_URI_form_bad(self):
1894         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
1895                              "400 Bad Request", "GET /uri requires uri=",
1896                              self.GET, "/uri")
1897         return d
1898
1899     def test_GET_rename_form(self):
1900         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
1901                      followRedirect=True)
1902         def _check(res):
1903             self.failUnless('name="when_done" value="."' in res, res)
1904             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
1905         d.addCallback(_check)
1906         return d
1907
1908     def log(self, res, msg):
1909         #print "MSG: %s  RES: %s" % (msg, res)
1910         log.msg(msg)
1911         return res
1912
1913     def test_GET_URI_URL(self):
1914         base = "/uri/%s" % self._bar_txt_uri
1915         d = self.GET(base)
1916         d.addCallback(self.failUnlessIsBarDotTxt)
1917         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
1918         d.addCallback(self.failUnlessIsBarDotTxt)
1919         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
1920         d.addCallback(self.failUnlessIsBarDotTxt)
1921         return d
1922
1923     def test_GET_URI_URL_dir(self):
1924         base = "/uri/%s?t=json" % self._foo_uri
1925         d = self.GET(base)
1926         d.addCallback(self.failUnlessIsFooJSON)
1927         return d
1928
1929     def test_GET_URI_URL_missing(self):
1930         base = "/uri/%s" % self._bad_file_uri
1931         d = self.GET(base)
1932         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
1933                   http.GONE, response_substring="NotEnoughSharesError")
1934         # TODO: how can we exercise both sides of WebDownloadTarget.fail
1935         # here? we must arrange for a download to fail after target.open()
1936         # has been called, and then inspect the response to see that it is
1937         # shorter than we expected.
1938         return d
1939
1940     def test_PUT_NEWFILEURL_uri(self):
1941         contents, n, new_uri = self.makefile(8)
1942         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
1943         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1944         d.addCallback(lambda res:
1945                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1946                                                       contents))
1947         return d
1948
1949     def test_PUT_NEWFILEURL_uri_replace(self):
1950         contents, n, new_uri = self.makefile(8)
1951         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
1952         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1953         d.addCallback(lambda res:
1954                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1955                                                       contents))
1956         return d
1957
1958     def test_PUT_NEWFILEURL_uri_no_replace(self):
1959         contents, n, new_uri = self.makefile(8)
1960         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
1961         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
1962                   "409 Conflict",
1963                   "There was already a child by that name, and you asked me "
1964                   "to not replace it")
1965         return d
1966
1967     def test_PUT_NEWFILE_URI(self):
1968         file_contents = "New file contents here\n"
1969         d = self.PUT("/uri", file_contents)
1970         def _check(uri):
1971             self.failUnless(uri in FakeCHKFileNode.all_contents)
1972             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
1973                                  file_contents)
1974             return self.GET("/uri/%s" % uri)
1975         d.addCallback(_check)
1976         def _check2(res):
1977             self.failUnlessEqual(res, file_contents)
1978         d.addCallback(_check2)
1979         return d
1980
1981     def test_PUT_NEWFILE_URI_only_PUT(self):
1982         d = self.PUT("/uri?t=bogus", "")
1983         d.addBoth(self.shouldFail, error.Error,
1984                   "PUT_NEWFILE_URI_only_PUT",
1985                   "400 Bad Request",
1986                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
1987         return d
1988
1989     def test_PUT_NEWFILE_URI_mutable(self):
1990         file_contents = "New file contents here\n"
1991         d = self.PUT("/uri?mutable=true", file_contents)
1992         def _check_mutable(uri):
1993             uri = uri.strip()
1994             u = IURI(uri)
1995             self.failUnless(IMutableFileURI.providedBy(u))
1996             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1997             n = self.s.create_node_from_uri(uri)
1998             return n.download_best_version()
1999         d.addCallback(_check_mutable)
2000         def _check2_mutable(data):
2001             self.failUnlessEqual(data, file_contents)
2002         d.addCallback(_check2_mutable)
2003         return d
2004
2005         def _check(uri):
2006             self.failUnless(uri in FakeCHKFileNode.all_contents)
2007             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2008                                  file_contents)
2009             return self.GET("/uri/%s" % uri)
2010         d.addCallback(_check)
2011         def _check2(res):
2012             self.failUnlessEqual(res, file_contents)
2013         d.addCallback(_check2)
2014         return d
2015
2016     def test_PUT_mkdir(self):
2017         d = self.PUT("/uri?t=mkdir", "")
2018         def _check(uri):
2019             n = self.s.create_node_from_uri(uri.strip())
2020             d2 = self.failUnlessNodeKeysAre(n, [])
2021             d2.addCallback(lambda res:
2022                            self.GET("/uri/%s?t=json" % uri))
2023             return d2
2024         d.addCallback(_check)
2025         d.addCallback(self.failUnlessIsEmptyJSON)
2026         return d
2027
2028     def test_POST_check(self):
2029         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2030         def _done(res):
2031             # this returns a string form of the results, which are probably
2032             # None since we're using fake filenodes.
2033             # TODO: verify that the check actually happened, by changing
2034             # FakeCHKFileNode to count how many times .check() has been
2035             # called.
2036             pass
2037         d.addCallback(_done)
2038         return d
2039
2040     def test_bad_method(self):
2041         url = self.webish_url + self.public_url + "/foo/bar.txt"
2042         d = self.shouldHTTPError2("test_bad_method",
2043                                   501, "Not Implemented",
2044                                   "I don't know how to treat a BOGUS request.",
2045                                   client.getPage, url, method="BOGUS")
2046         return d
2047
2048     def test_short_url(self):
2049         url = self.webish_url + "/uri"
2050         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2051                                   "I don't know how to treat a DELETE request.",
2052                                   client.getPage, url, method="DELETE")
2053         return d
2054
2055 class Util(unittest.TestCase):
2056     def test_abbreviate_time(self):
2057         self.failUnlessEqual(common.abbreviate_time(None), "")
2058         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2059         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2060         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2061         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2062
2063     def test_abbreviate_rate(self):
2064         self.failUnlessEqual(common.abbreviate_rate(None), "")
2065         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2066         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2067         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2068
2069     def test_abbreviate_size(self):
2070         self.failUnlessEqual(common.abbreviate_size(None), "")
2071         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2072         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2073         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2074         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2075