src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17
  18 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  19 # create a webserver that works against them
  20
  21 class FakeIntroducerClient:
  22     def get_all_connectors(self):
  23         return {}
  24     def get_all_connections_for(self, service_name):
  25         return frozenset()
  26     def get_all_peerids(self):
  27         return frozenset()
  28
  29 class FakeClient(service.MultiService):
  30     nodeid = "fake_nodeid"
  31     nickname = "fake_nickname"
  32     basedir = "fake_basedir"
  33     def get_versions(self):
  34         return {'allmydata': "fake",
  35                 'foolscap': "fake",
  36                 'twisted': "fake",
  37                 'zfec': "fake",
  38                 }
  39     introducer_furl = "None"
  40     introducer_client = FakeIntroducerClient()
  41     _all_upload_status = [upload.UploadStatus()]
  42     _all_download_status = [download.DownloadStatus()]
  43     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  44     _all_publish_statuses = [publish.PublishStatus()]
  45     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  46     convergence = "some random string"
  47
  48     def connected_to_introducer(self):
  49         return False
  50
  51     def create_node_from_uri(self, auri):
  52         u = uri.from_string(auri)
  53         if (INewDirectoryURI.providedBy(u)
  54             or IReadonlyNewDirectoryURI.providedBy(u)):
  55             return FakeDirectoryNode(self).init_from_uri(u)
  56         if IFileURI.providedBy(u):
  57             return FakeCHKFileNode(u, self)
  58         assert IMutableFileURI.providedBy(u), u
  59         return FakeMutableFileNode(self).init_from_uri(u)
  60
  61     def create_empty_dirnode(self):
  62         n = FakeDirectoryNode(self)
  63         d = n.create()
  64         d.addCallback(lambda res: n)
  65         return d
  66
  67     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  68     def create_mutable_file(self, contents=""):
  69         n = FakeMutableFileNode(self)
  70         return n.create(contents)
  71
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(self, data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83
  84     def list_all_upload_statuses(self):
  85         return self._all_upload_status
  86     def list_all_download_statuses(self):
  87         return self._all_download_status
  88     def list_all_mapupdate_statuses(self):
  89         return self._all_mapupdate_statuses
  90     def list_all_publish_statuses(self):
  91         return self._all_publish_statuses
  92     def list_all_retrieve_statuses(self):
  93         return self._all_retrieve_statuses
  94     def list_all_helper_statuses(self):
  95         return []
  96
  97 class HTTPClientHEADFactory(client.HTTPClientFactory):
  98     def __init__(self, *args, **kwargs):
  99         client.HTTPClientFactory.__init__(self, *args, **kwargs)
 100         self.deferred.addCallback(lambda res: self.response_headers)
 101
 102
 103 class WebMixin(object):
 104     def setUp(self):
 105         self.s = FakeClient()
 106         self.s.startService()
 107         self.ws = s = webish.WebishServer("0")
 108         s.setServiceParent(self.s)
 109         self.webish_port = port = s.listener._port.getHost().port
 110         self.webish_url = "http://localhost:%d" % port
 111
 112         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 113         d = defer.DeferredList(l)
 114         def _then(res):
 115             self.public_root = res[0][1]
 116             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 117             self.public_url = "/uri/" + self.public_root.get_uri()
 118             self.private_root = res[1][1]
 119
 120             foo = res[2][1]
 121             self._foo_node = foo
 122             self._foo_uri = foo.get_uri()
 123             self._foo_readonly_uri = foo.get_readonly_uri()
 124             # NOTE: we ignore the deferred on all set_uri() calls, because we
 125             # know the fake nodes do these synchronously
 126             self.public_root.set_uri(u"foo", foo.get_uri())
 127
 128             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 129             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 130
 131             foo.set_uri(u"empty", res[3][1].get_uri())
 132             sub_uri = res[4][1].get_uri()
 133             self._sub_uri = sub_uri
 134             foo.set_uri(u"sub", sub_uri)
 135             sub = self.s.create_node_from_uri(sub_uri)
 136
 137             _ign, n, blocking_uri = self.makefile(1)
 138             foo.set_uri(u"blockingfile", blocking_uri)
 139
 140             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 141             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 142             # still think of it as an umlaut
 143             foo.set_uri(unicode_filename, self._bar_txt_uri)
 144
 145             _ign, n, baz_file = self.makefile(2)
 146             sub.set_uri(u"baz.txt", baz_file)
 147
 148             _ign, n, self._bad_file_uri = self.makefile(3)
 149             # this uri should not be downloadable
 150             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 151
 152             rodir = res[5][1]
 153             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 154             rodir.set_uri(u"nor", baz_file)
 155
 156             # public/
 157             # public/foo/
 158             # public/foo/bar.txt
 159             # public/foo/blockingfile
 160             # public/foo/empty/
 161             # public/foo/sub/
 162             # public/foo/sub/baz.txt
 163             # public/reedownlee/
 164             # public/reedownlee/nor
 165             self.NEWFILE_CONTENTS = "newfile contents\n"
 166
 167             return foo.get_metadata_for(u"bar.txt")
 168         d.addCallback(_then)
 169         def _got_metadata(metadata):
 170             self._bar_txt_metadata = metadata
 171         d.addCallback(_got_metadata)
 172         return d
 173
 174     def makefile(self, number):
 175         contents = "contents of file %s\n" % number
 176         n = create_chk_filenode(self.s, contents)
 177         return contents, n, n.get_uri()
 178
 179     def tearDown(self):
 180         return self.s.stopService()
 181
 182     def failUnlessIsBarDotTxt(self, res):
 183         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 184
 185     def failUnlessIsBarJSON(self, res):
 186         data = simplejson.loads(res)
 187         self.failUnless(isinstance(data, list))
 188         self.failUnlessEqual(data[0], "filenode")
 189         self.failUnless(isinstance(data[1], dict))
 190         self.failIf(data[1]["mutable"])
 191         self.failIf("rw_uri" in data[1]) # immutable
 192         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 193         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 194
 195     def failUnlessIsFooJSON(self, res):
 196         data = simplejson.loads(res)
 197         self.failUnless(isinstance(data, list))
 198         self.failUnlessEqual(data[0], "dirnode", res)
 199         self.failUnless(isinstance(data[1], dict))
 200         self.failUnless(data[1]["mutable"])
 201         self.failUnless("rw_uri" in data[1]) # mutable
 202         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 203         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 204
 205         kidnames = sorted(data[1]["children"])
 206         self.failUnlessEqual(kidnames,
 207                              [u"bar.txt", u"blockingfile", u"empty",
 208                               u"n\u00fc.txt", u"sub"])
 209         kids = data[1]["children"]
 210         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 211         self.failUnless("metadata" in kids[u"sub"][1])
 212         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 213         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 214         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 215         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 216         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 217         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 218                              self._bar_txt_metadata["ctime"])
 219         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 220                              self._bar_txt_uri)
 221
 222     def GET(self, urlpath, followRedirect=False):
 223         assert not isinstance(urlpath, unicode)
 224         url = self.webish_url + urlpath
 225         return client.getPage(url, method="GET", followRedirect=followRedirect)
 226
 227     def HEAD(self, urlpath):
 228         # this requires some surgery, because twisted.web.client doesn't want
 229         # to give us back the response headers.
 230         factory = HTTPClientHEADFactory(urlpath)
 231         reactor.connectTCP("localhost", self.webish_port, factory)
 232         return factory.deferred
 233
 234     def PUT(self, urlpath, data):
 235         url = self.webish_url + urlpath
 236         return client.getPage(url, method="PUT", postdata=data)
 237
 238     def DELETE(self, urlpath):
 239         url = self.webish_url + urlpath
 240         return client.getPage(url, method="DELETE")
 241
 242     def POST(self, urlpath, followRedirect=False, **fields):
 243         url = self.webish_url + urlpath
 244         sepbase = "boogabooga"
 245         sep = "--" + sepbase
 246         form = []
 247         form.append(sep)
 248         form.append('Content-Disposition: form-data; name="_charset"')
 249         form.append('')
 250         form.append('UTF-8')
 251         form.append(sep)
 252         for name, value in fields.iteritems():
 253             if isinstance(value, tuple):
 254                 filename, value = value
 255                 form.append('Content-Disposition: form-data; name="%s"; '
 256                             'filename="%s"' % (name, filename.encode("utf-8")))
 257             else:
 258                 form.append('Content-Disposition: form-data; name="%s"' % name)
 259             form.append('')
 260             if isinstance(value, unicode):
 261                 value = value.encode("utf-8")
 262             else:
 263                 value = str(value)
 264             assert isinstance(value, str)
 265             form.append(value)
 266             form.append(sep)
 267         form[-1] += "--"
 268         body = "\r\n".join(form) + "\r\n"
 269         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 270                    }
 271         return client.getPage(url, method="POST", postdata=body,
 272                               headers=headers, followRedirect=followRedirect)
 273
 274     def shouldFail(self, res, expected_failure, which,
 275                    substring=None, response_substring=None):
 276         if isinstance(res, failure.Failure):
 277             res.trap(expected_failure)
 278             if substring:
 279                 self.failUnless(substring in str(res),
 280                                 "substring '%s' not in '%s'"
 281                                 % (substring, str(res)))
 282             if response_substring:
 283                 self.failUnless(response_substring in res.value.response,
 284                                 "response substring '%s' not in '%s'"
 285                                 % (response_substring, res.value.response))
 286         else:
 287             self.fail("%s was supposed to raise %s, not get '%s'" %
 288                       (which, expected_failure, res))
 289
 290     def shouldFail2(self, expected_failure, which, substring,
 291                     response_substring,
 292                     callable, *args, **kwargs):
 293         assert substring is None or isinstance(substring, str)
 294         assert response_substring is None or isinstance(response_substring, str)
 295         d = defer.maybeDeferred(callable, *args, **kwargs)
 296         def done(res):
 297             if isinstance(res, failure.Failure):
 298                 res.trap(expected_failure)
 299                 if substring:
 300                     self.failUnless(substring in str(res),
 301                                     "%s: substring '%s' not in '%s'"
 302                                     % (which, substring, str(res)))
 303                 if response_substring:
 304                     self.failUnless(response_substring in res.value.response,
 305                                     "%s: response substring '%s' not in '%s'"
 306                                     % (which,
 307                                        response_substring, res.value.response))
 308             else:
 309                 self.fail("%s was supposed to raise %s, not get '%s'" %
 310                           (which, expected_failure, res))
 311         d.addBoth(done)
 312         return d
 313
 314     def should404(self, res, which):
 315         if isinstance(res, failure.Failure):
 316             res.trap(error.Error)
 317             self.failUnlessEqual(res.value.status, "404")
 318         else:
 319             self.fail("%s was supposed to Error(404), not get '%s'" %
 320                       (which, res))
 321
 322     def shouldHTTPError(self, res, which, code=None, substring=None,
 323                         response_substring=None):
 324         if isinstance(res, failure.Failure):
 325             res.trap(error.Error)
 326             if code is not None:
 327                 self.failUnlessEqual(res.value.status, str(code))
 328             if substring:
 329                 self.failUnless(substring in str(res),
 330                                 "substring '%s' not in '%s'"
 331                                 % (substring, str(res)))
 332             if response_substring:
 333                 self.failUnless(response_substring in res.value.response,
 334                                 "response substring '%s' not in '%s'"
 335                                 % (response_substring, res.value.response))
 336         else:
 337             self.fail("%s was supposed to Error(%s), not get '%s'" %
 338                       (which, code, res))
 339
 340     def shouldHTTPError2(self, which,
 341                          code=None, substring=None, response_substring=None,
 342                          callable=None, *args, **kwargs):
 343         assert substring is None or isinstance(substring, str)
 344         assert callable
 345         d = defer.maybeDeferred(callable, *args, **kwargs)
 346         d.addBoth(self.shouldHTTPError, which,
 347                   code, substring, response_substring)
 348         return d
 349
 350
 351 class Web(WebMixin, unittest.TestCase):
 352     def test_create(self):
 353         pass
 354
 355     def test_welcome(self):
 356         d = self.GET("/")
 357         def _check(res):
 358             self.failUnless('Welcome To AllMyData' in res)
 359             self.failUnless('Tahoe' in res)
 360
 361             self.s.basedir = 'web/test_welcome'
 362             fileutil.make_dirs("web/test_welcome")
 363             fileutil.make_dirs("web/test_welcome/private")
 364             return self.GET("/")
 365         d.addCallback(_check)
 366         return d
 367
 368     def test_provisioning_math(self):
 369         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 370         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 371         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 372         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 373         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 374
 375     def test_provisioning(self):
 376         d = self.GET("/provisioning/")
 377         def _check(res):
 378             self.failUnless('Tahoe Provisioning Tool' in res)
 379             fields = {'filled': True,
 380                       "num_users": int(50e3),
 381                       "files_per_user": 1000,
 382                       "space_per_user": int(1e9),
 383                       "sharing_ratio": 1.0,
 384                       "encoding_parameters": "3-of-10-5",
 385                       "num_servers": 30,
 386                       "ownership_mode": "A",
 387                       "download_rate": 100,
 388                       "upload_rate": 10,
 389                       "delete_rate": 10,
 390                       "lease_timer": 7,
 391                       }
 392             return self.POST("/provisioning/", **fields)
 393
 394         d.addCallback(_check)
 395         def _check2(res):
 396             self.failUnless('Tahoe Provisioning Tool' in res)
 397             self.failUnless("Share space consumed: 167.01TB" in res)
 398
 399             fields = {'filled': True,
 400                       "num_users": int(50e6),
 401                       "files_per_user": 1000,
 402                       "space_per_user": int(5e9),
 403                       "sharing_ratio": 1.0,
 404                       "encoding_parameters": "25-of-100-50",
 405                       "num_servers": 30000,
 406                       "ownership_mode": "E",
 407                       "drive_failure_model": "U",
 408                       "drive_size": 1000,
 409                       "download_rate": 1000,
 410                       "upload_rate": 100,
 411                       "delete_rate": 100,
 412                       "lease_timer": 7,
 413                       }
 414             return self.POST("/provisioning/", **fields)
 415         d.addCallback(_check2)
 416         def _check3(res):
 417             self.failUnless("Share space consumed: huge!" in res)
 418             fields = {'filled': True}
 419             return self.POST("/provisioning/", **fields)
 420         d.addCallback(_check3)
 421         def _check4(res):
 422             self.failUnless("Share space consumed:" in res)
 423         d.addCallback(_check4)
 424         return d
 425
 426     def test_status(self):
 427         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 428         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 429         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 430         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 431         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 432         d = self.GET("/status", followRedirect=True)
 433         def _check(res):
 434             self.failUnless('Upload and Download Status' in res, res)
 435             self.failUnless('"down-%d"' % dl_num in res, res)
 436             self.failUnless('"up-%d"' % ul_num in res, res)
 437             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 438             self.failUnless('"publish-%d"' % pub_num in res, res)
 439             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 440         d.addCallback(_check)
 441         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 442         def _check_dl(res):
 443             self.failUnless("File Download Status" in res, res)
 444         d.addCallback(_check_dl)
 445         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 446         def _check_ul(res):
 447             self.failUnless("File Upload Status" in res, res)
 448         d.addCallback(_check_ul)
 449         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 450         def _check_mapupdate(res):
 451             self.failUnless("Mutable File Servermap Update Status" in res, res)
 452         d.addCallback(_check_mapupdate)
 453         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 454         def _check_publish(res):
 455             self.failUnless("Mutable File Publish Status" in res, res)
 456         d.addCallback(_check_publish)
 457         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 458         def _check_retrieve(res):
 459             self.failUnless("Mutable File Retrieve Status" in res, res)
 460         d.addCallback(_check_retrieve)
 461
 462         return d
 463
 464     def test_status_numbers(self):
 465         drrm = status.DownloadResultsRendererMixin()
 466         self.failUnlessEqual(drrm.render_time(None, None), "")
 467         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 468         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 469         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 470         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 471         self.failUnlessEqual(drrm.render_rate(None, None), "")
 472         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 473         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 474         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 475
 476         urrm = status.UploadResultsRendererMixin()
 477         self.failUnlessEqual(urrm.render_time(None, None), "")
 478         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 479         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 480         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 481         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 482         self.failUnlessEqual(urrm.render_rate(None, None), "")
 483         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 484         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 485         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 486
 487     def test_GET_FILEURL(self):
 488         d = self.GET(self.public_url + "/foo/bar.txt")
 489         d.addCallback(self.failUnlessIsBarDotTxt)
 490         return d
 491
 492     def test_HEAD_FILEURL(self):
 493         d = self.HEAD(self.public_url + "/foo/bar.txt")
 494         def _got(headers):
 495             self.failUnlessEqual(headers["content-length"][0],
 496                                  str(len(self.BAR_CONTENTS)))
 497             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 498         d.addCallback(_got)
 499         return d
 500
 501     def test_GET_FILEURL_named(self):
 502         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 503         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 504         d = self.GET(base + "/@@name=/blah.txt")
 505         d.addCallback(self.failUnlessIsBarDotTxt)
 506         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 507         d.addCallback(self.failUnlessIsBarDotTxt)
 508         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 509         d.addCallback(self.failUnlessIsBarDotTxt)
 510         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 511         d.addCallback(self.failUnlessIsBarDotTxt)
 512         save_url = base + "?save=true&filename=blah.txt"
 513         d.addCallback(lambda res: self.GET(save_url))
 514         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 515         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 516         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 517         u_url = base + "?save=true&filename=" + u_fn_e
 518         d.addCallback(lambda res: self.GET(u_url))
 519         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 520         return d
 521
 522     def test_PUT_FILEURL_named_bad(self):
 523         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 524         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 525                              "400 Bad Request",
 526                              "/file can only be used with GET or HEAD",
 527                              self.PUT, base + "/@@name=/blah.txt", "")
 528         return d
 529
 530     def test_GET_DIRURL_named_bad(self):
 531         base = "/file/%s" % urllib.quote(self._foo_uri)
 532         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 533                              "400 Bad Request",
 534                              "is not a file-cap",
 535                              self.GET, base + "/@@name=/blah.txt")
 536         return d
 537
 538     def test_GET_slash_file_bad(self):
 539         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 540                              "404 Not Found",
 541                              "/file must be followed by a file-cap and a name",
 542                              self.GET, "/file")
 543         return d
 544
 545     def test_GET_unhandled_URI_named(self):
 546         contents, n, newuri = self.makefile(12)
 547         verifier_cap = n.get_verifier().to_string()
 548         base = "/file/%s" % urllib.quote(verifier_cap)
 549         # client.create_node_from_uri() can't handle verify-caps
 550         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 551                              "400 Bad Request",
 552                              "is not a valid file- or directory- cap",
 553                              self.GET, base)
 554         return d
 555
 556     def test_GET_unhandled_URI(self):
 557         contents, n, newuri = self.makefile(12)
 558         verifier_cap = n.get_verifier().to_string()
 559         base = "/uri/%s" % urllib.quote(verifier_cap)
 560         # client.create_node_from_uri() can't handle verify-caps
 561         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 562                              "400 Bad Request",
 563                              "is not a valid file- or directory- cap",
 564                              self.GET, base)
 565         return d
 566
 567     def test_GET_FILE_URI(self):
 568         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 569         d = self.GET(base)
 570         d.addCallback(self.failUnlessIsBarDotTxt)
 571         return d
 572
 573     def test_GET_FILE_URI_badchild(self):
 574         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 575         errmsg = "Files have no children, certainly not named 'boguschild'"
 576         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 577                              "400 Bad Request", errmsg,
 578                              self.GET, base)
 579         return d
 580
 581     def test_PUT_FILE_URI_badchild(self):
 582         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 583         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 584         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 585                              "400 Bad Request", errmsg,
 586                              self.PUT, base, "")
 587         return d
 588
 589     def test_GET_FILEURL_save(self):
 590         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 591         # TODO: look at the headers, expect a Content-Disposition: attachment
 592         # header.
 593         d.addCallback(self.failUnlessIsBarDotTxt)
 594         return d
 595
 596     def test_GET_FILEURL_missing(self):
 597         d = self.GET(self.public_url + "/foo/missing")
 598         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 599         return d
 600
 601     def test_PUT_NEWFILEURL(self):
 602         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 603         # TODO: we lose the response code, so we can't check this
 604         #self.failUnlessEqual(responsecode, 201)
 605         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 606         d.addCallback(lambda res:
 607                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 608                                                       self.NEWFILE_CONTENTS))
 609         return d
 610
 611     def test_PUT_NEWFILEURL_mutable(self):
 612         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 613                      self.NEWFILE_CONTENTS)
 614         # TODO: we lose the response code, so we can't check this
 615         #self.failUnlessEqual(responsecode, 201)
 616         def _check_uri(res):
 617             u = uri.from_string_mutable_filenode(res)
 618             self.failUnless(u.is_mutable())
 619             self.failIf(u.is_readonly())
 620             return res
 621         d.addCallback(_check_uri)
 622         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 623         d.addCallback(lambda res:
 624                       self.failUnlessMutableChildContentsAre(self._foo_node,
 625                                                              u"new.txt",
 626                                                              self.NEWFILE_CONTENTS))
 627         return d
 628
 629     def test_PUT_NEWFILEURL_mutable_toobig(self):
 630         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 631                              "413 Request Entity Too Large",
 632                              "SDMF is limited to one segment, and 10001 > 10000",
 633                              self.PUT,
 634                              self.public_url + "/foo/new.txt?mutable=true",
 635                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 636         return d
 637
 638     def test_PUT_NEWFILEURL_replace(self):
 639         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 640         # TODO: we lose the response code, so we can't check this
 641         #self.failUnlessEqual(responsecode, 200)
 642         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 643         d.addCallback(lambda res:
 644                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 645                                                       self.NEWFILE_CONTENTS))
 646         return d
 647
 648     def test_PUT_NEWFILEURL_bad_t(self):
 649         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 650                              "PUT to a file: bad t=bogus",
 651                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 652                              "contents")
 653         return d
 654
 655     def test_PUT_NEWFILEURL_no_replace(self):
 656         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 657                      self.NEWFILE_CONTENTS)
 658         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 659                   "409 Conflict",
 660                   "There was already a child by that name, and you asked me "
 661                   "to not replace it")
 662         return d
 663
 664     def test_PUT_NEWFILEURL_mkdirs(self):
 665         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 666         fn = self._foo_node
 667         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 668         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 669         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 670         d.addCallback(lambda res:
 671                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 672                                                       self.NEWFILE_CONTENTS))
 673         return d
 674
 675     def test_PUT_NEWFILEURL_blocked(self):
 676         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 677                      self.NEWFILE_CONTENTS)
 678         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 679                   "409 Conflict",
 680                   "Unable to create directory 'blockingfile': a file was in the way")
 681         return d
 682
 683     def test_DELETE_FILEURL(self):
 684         d = self.DELETE(self.public_url + "/foo/bar.txt")
 685         d.addCallback(lambda res:
 686                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 687         return d
 688
 689     def test_DELETE_FILEURL_missing(self):
 690         d = self.DELETE(self.public_url + "/foo/missing")
 691         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 692         return d
 693
 694     def test_DELETE_FILEURL_missing2(self):
 695         d = self.DELETE(self.public_url + "/missing/missing")
 696         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 697         return d
 698
 699     def test_GET_FILEURL_json(self):
 700         # twisted.web.http.parse_qs ignores any query args without an '=', so
 701         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 702         # instead. This may make it tricky to emulate the S3 interface
 703         # completely.
 704         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 705         d.addCallback(self.failUnlessIsBarJSON)
 706         return d
 707
 708     def test_GET_FILEURL_json_missing(self):
 709         d = self.GET(self.public_url + "/foo/missing?json")
 710         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 711         return d
 712
 713     def test_GET_FILEURL_uri(self):
 714         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 715         def _check(res):
 716             self.failUnlessEqual(res, self._bar_txt_uri)
 717         d.addCallback(_check)
 718         d.addCallback(lambda res:
 719                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 720         def _check2(res):
 721             # for now, for files, uris and readonly-uris are the same
 722             self.failUnlessEqual(res, self._bar_txt_uri)
 723         d.addCallback(_check2)
 724         return d
 725
 726     def test_GET_FILEURL_badtype(self):
 727         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 728                                   "bad t=bogus",
 729                                   self.GET,
 730                                   self.public_url + "/foo/bar.txt?t=bogus")
 731         return d
 732
 733     def test_GET_FILEURL_uri_missing(self):
 734         d = self.GET(self.public_url + "/foo/missing?t=uri")
 735         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 736         return d
 737
 738     def test_GET_DIRURL(self):
 739         # the addSlash means we get a redirect here
 740         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 741         ROOT = "../../.."
 742         d = self.GET(self.public_url + "/foo", followRedirect=True)
 743         def _check(res):
 744             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 745                             in res, res)
 746             # the FILE reference points to a URI, but it should end in bar.txt
 747             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 748                        (ROOT, urllib.quote(self._bar_txt_uri)))
 749             get_bar = "".join([r'<td>',
 750                                r'<a href="%s">bar.txt</a>' % bar_url,
 751                                r'</td>',
 752                                r'\s+<td>FILE</td>',
 753                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 754                                ])
 755             self.failUnless(re.search(get_bar, res), res)
 756             for line in res.split("\n"):
 757                 # find the line that contains the delete button for bar.txt
 758                 if ("form action" in line and
 759                     'value="delete"' in line and
 760                     'value="bar.txt"' in line):
 761                     # the form target should use a relative URL
 762                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 763                     self.failUnless(('action="%s"' % foo_url) in line, line)
 764                     # and the when_done= should too
 765                     #done_url = urllib.quote(???)
 766                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 767                     #                in line, line)
 768                     break
 769             else:
 770                 self.fail("unable to find delete-bar.txt line", res)
 771
 772             # the DIR reference just points to a URI
 773             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 774             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 775                        + r'\s+<td>DIR</td>')
 776             self.failUnless(re.search(get_sub, res), res)
 777         d.addCallback(_check)
 778
 779         # look at a directory which is readonly
 780         d.addCallback(lambda res:
 781                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 782         def _check2(res):
 783             self.failUnless("(readonly)" in res, res)
 784             self.failIf("Upload a file" in res, res)
 785         d.addCallback(_check2)
 786
 787         # and at a directory that contains a readonly directory
 788         d.addCallback(lambda res:
 789                       self.GET(self.public_url, followRedirect=True))
 790         def _check3(res):
 791             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 792                                       '</td>\s+<td>DIR-RO</td>', res))
 793         d.addCallback(_check3)
 794
 795         return d
 796
 797     def test_GET_DIRURL_badtype(self):
 798         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 799                                   400, "Bad Request",
 800                                   "bad t=bogus",
 801                                   self.GET,
 802                                   self.public_url + "/foo?t=bogus")
 803         return d
 804
 805     def test_GET_DIRURL_json(self):
 806         d = self.GET(self.public_url + "/foo?t=json")
 807         d.addCallback(self.failUnlessIsFooJSON)
 808         return d
 809
 810     def test_GET_DIRURL_manifest(self):
 811         d = self.GET(self.public_url + "/foo?t=manifest", followRedirect=True)
 812         def _got(manifest):
 813             self.failUnless("Refresh Capabilities" in manifest)
 814         d.addCallback(_got)
 815         return d
 816
 817     def test_GET_DIRURL_deepsize(self):
 818         d = self.GET(self.public_url + "/foo?t=deep-size", followRedirect=True)
 819         def _got(manifest):
 820             self.failUnless(re.search(r'^\d+$', manifest), manifest)
 821             self.failUnlessEqual("57", manifest)
 822         d.addCallback(_got)
 823         return d
 824
 825     def test_GET_DIRURL_deepstats(self):
 826         d = self.GET(self.public_url + "/foo?t=deep-stats", followRedirect=True)
 827         def _got(stats_json):
 828             stats = simplejson.loads(stats_json)
 829             expected = {"count-immutable-files": 3,
 830                         "count-mutable-files": 0,
 831                         "count-literal-files": 0,
 832                         "count-files": 3,
 833                         "count-directories": 3,
 834                         "size-immutable-files": 57,
 835                         "size-literal-files": 0,
 836                         #"size-directories": 1912, # varies
 837                         #"largest-directory": 1590,
 838                         "largest-directory-children": 5,
 839                         "largest-immutable-file": 19,
 840                         }
 841             for k,v in expected.iteritems():
 842                 self.failUnlessEqual(stats[k], v,
 843                                      "stats[%s] was %s, not %s" %
 844                                      (k, stats[k], v))
 845             self.failUnlessEqual(stats["size-files-histogram"],
 846                                  [ [11, 31, 3] ])
 847         d.addCallback(_got)
 848         return d
 849
 850     def test_GET_DIRURL_uri(self):
 851         d = self.GET(self.public_url + "/foo?t=uri")
 852         def _check(res):
 853             self.failUnlessEqual(res, self._foo_uri)
 854         d.addCallback(_check)
 855         return d
 856
 857     def test_GET_DIRURL_readonly_uri(self):
 858         d = self.GET(self.public_url + "/foo?t=readonly-uri")
 859         def _check(res):
 860             self.failUnlessEqual(res, self._foo_readonly_uri)
 861         d.addCallback(_check)
 862         return d
 863
 864     def test_PUT_NEWDIRURL(self):
 865         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
 866         d.addCallback(lambda res:
 867                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
 868         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
 869         d.addCallback(self.failUnlessNodeKeysAre, [])
 870         return d
 871
 872     def test_PUT_NEWDIRURL_exists(self):
 873         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
 874         d.addCallback(lambda res:
 875                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 876         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 877         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 878         return d
 879
 880     def test_PUT_NEWDIRURL_blocked(self):
 881         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
 882                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
 883                              self.PUT,
 884                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
 885         d.addCallback(lambda res:
 886                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 887         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 888         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 889         return d
 890
 891     def test_PUT_NEWDIRURL_mkdir_p(self):
 892         d = defer.succeed(None)
 893         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
 894         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
 895         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
 896         def mkdir_p(mkpnode):
 897             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
 898             d = self.POST(url)
 899             def made_subsub(ssuri):
 900                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
 901                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
 902                 d = self.POST(url)
 903                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
 904                 return d
 905             d.addCallback(made_subsub)
 906             return d
 907         d.addCallback(mkdir_p)
 908         return d
 909
 910     def test_PUT_NEWDIRURL_mkdirs(self):
 911         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
 912         d.addCallback(lambda res:
 913                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
 914         d.addCallback(lambda res:
 915                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
 916         d.addCallback(lambda res:
 917                       self._foo_node.get_child_at_path(u"subdir/newdir"))
 918         d.addCallback(self.failUnlessNodeKeysAre, [])
 919         return d
 920
 921     def test_DELETE_DIRURL(self):
 922         d = self.DELETE(self.public_url + "/foo")
 923         d.addCallback(lambda res:
 924                       self.failIfNodeHasChild(self.public_root, u"foo"))
 925         return d
 926
 927     def test_DELETE_DIRURL_missing(self):
 928         d = self.DELETE(self.public_url + "/foo/missing")
 929         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
 930         d.addCallback(lambda res:
 931                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
 932         return d
 933
 934     def test_DELETE_DIRURL_missing2(self):
 935         d = self.DELETE(self.public_url + "/missing")
 936         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
 937         return d
 938
 939     def dump_root(self):
 940         print "NODEWALK"
 941         w = webish.DirnodeWalkerMixin()
 942         def visitor(childpath, childnode, metadata):
 943             print childpath
 944         d = w.walk(self.public_root, visitor)
 945         return d
 946
 947     def failUnlessNodeKeysAre(self, node, expected_keys):
 948         for k in expected_keys:
 949             assert isinstance(k, unicode)
 950         d = node.list()
 951         def _check(children):
 952             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
 953         d.addCallback(_check)
 954         return d
 955     def failUnlessNodeHasChild(self, node, name):
 956         assert isinstance(name, unicode)
 957         d = node.list()
 958         def _check(children):
 959             self.failUnless(name in children)
 960         d.addCallback(_check)
 961         return d
 962     def failIfNodeHasChild(self, node, name):
 963         assert isinstance(name, unicode)
 964         d = node.list()
 965         def _check(children):
 966             self.failIf(name in children)
 967         d.addCallback(_check)
 968         return d
 969
 970     def failUnlessChildContentsAre(self, node, name, expected_contents):
 971         assert isinstance(name, unicode)
 972         d = node.get_child_at_path(name)
 973         d.addCallback(lambda node: node.download_to_data())
 974         def _check(contents):
 975             self.failUnlessEqual(contents, expected_contents)
 976         d.addCallback(_check)
 977         return d
 978
 979     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
 980         assert isinstance(name, unicode)
 981         d = node.get_child_at_path(name)
 982         d.addCallback(lambda node: node.download_best_version())
 983         def _check(contents):
 984             self.failUnlessEqual(contents, expected_contents)
 985         d.addCallback(_check)
 986         return d
 987
 988     def failUnlessChildURIIs(self, node, name, expected_uri):
 989         assert isinstance(name, unicode)
 990         d = node.get_child_at_path(name)
 991         def _check(child):
 992             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
 993         d.addCallback(_check)
 994         return d
 995
 996     def failUnlessURIMatchesChild(self, got_uri, node, name):
 997         assert isinstance(name, unicode)
 998         d = node.get_child_at_path(name)
 999         def _check(child):
1000             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1001         d.addCallback(_check)
1002         return d
1003
1004     def failUnlessCHKURIHasContents(self, got_uri, contents):
1005         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1006
1007     def test_POST_upload(self):
1008         d = self.POST(self.public_url + "/foo", t="upload",
1009                       file=("new.txt", self.NEWFILE_CONTENTS))
1010         fn = self._foo_node
1011         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1012         d.addCallback(lambda res:
1013                       self.failUnlessChildContentsAre(fn, u"new.txt",
1014                                                       self.NEWFILE_CONTENTS))
1015         return d
1016
1017     def test_POST_upload_unicode(self):
1018         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1019         d = self.POST(self.public_url + "/foo", t="upload",
1020                       file=(filename, self.NEWFILE_CONTENTS))
1021         fn = self._foo_node
1022         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1023         d.addCallback(lambda res:
1024                       self.failUnlessChildContentsAre(fn, filename,
1025                                                       self.NEWFILE_CONTENTS))
1026         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1027         d.addCallback(lambda res: self.GET(target_url))
1028         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1029                                                             self.NEWFILE_CONTENTS,
1030                                                             contents))
1031         return d
1032
1033     def test_POST_upload_unicode_named(self):
1034         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1035         d = self.POST(self.public_url + "/foo", t="upload",
1036                       name=filename,
1037                       file=("overridden", self.NEWFILE_CONTENTS))
1038         fn = self._foo_node
1039         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1040         d.addCallback(lambda res:
1041                       self.failUnlessChildContentsAre(fn, filename,
1042                                                       self.NEWFILE_CONTENTS))
1043         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1044         d.addCallback(lambda res: self.GET(target_url))
1045         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1046                                                             self.NEWFILE_CONTENTS,
1047                                                             contents))
1048         return d
1049
1050     def test_POST_upload_no_link(self):
1051         d = self.POST("/uri", t="upload",
1052                       file=("new.txt", self.NEWFILE_CONTENTS))
1053         def _check_upload_results(page):
1054             # this should be a page which describes the results of the upload
1055             # that just finished.
1056             self.failUnless("Upload Results:" in page)
1057             self.failUnless("URI:" in page)
1058             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1059             mo = uri_re.search(page)
1060             self.failUnless(mo, page)
1061             new_uri = mo.group(1)
1062             return new_uri
1063         d.addCallback(_check_upload_results)
1064         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1065         return d
1066
1067     def test_POST_upload_no_link_whendone(self):
1068         d = self.POST("/uri", t="upload", when_done="/",
1069                       file=("new.txt", self.NEWFILE_CONTENTS))
1070         d.addBoth(self.shouldRedirect, "/")
1071         return d
1072
1073     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1074         d = defer.maybeDeferred(callable, *args, **kwargs)
1075         def done(res):
1076             if isinstance(res, failure.Failure):
1077                 res.trap(error.PageRedirect)
1078                 statuscode = res.value.status
1079                 target = res.value.location
1080                 return checker(statuscode, target)
1081             self.fail("%s: callable was supposed to redirect, not return '%s'"
1082                       % (which, res))
1083         d.addBoth(done)
1084         return d
1085
1086     def test_POST_upload_no_link_whendone_results(self):
1087         def check(statuscode, target):
1088             self.failUnlessEqual(statuscode, str(http.FOUND))
1089             self.failUnless(target.startswith(self.webish_url), target)
1090             return client.getPage(target, method="GET")
1091         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1092                                  check,
1093                                  self.POST, "/uri", t="upload",
1094                                  when_done="/uri/%(uri)s",
1095                                  file=("new.txt", self.NEWFILE_CONTENTS))
1096         d.addCallback(lambda res:
1097                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1098         return d
1099
1100     def test_POST_upload_no_link_mutable(self):
1101         d = self.POST("/uri", t="upload", mutable="true",
1102                       file=("new.txt", self.NEWFILE_CONTENTS))
1103         def _check(new_uri):
1104             new_uri = new_uri.strip()
1105             self.new_uri = new_uri
1106             u = IURI(new_uri)
1107             self.failUnless(IMutableFileURI.providedBy(u))
1108             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1109             n = self.s.create_node_from_uri(new_uri)
1110             return n.download_best_version()
1111         d.addCallback(_check)
1112         def _check2(data):
1113             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1114             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1115         d.addCallback(_check2)
1116         def _check3(data):
1117             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1118             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1119         d.addCallback(_check3)
1120         def _check4(data):
1121             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1122         d.addCallback(_check4)
1123         return d
1124
1125     def test_POST_upload_no_link_mutable_toobig(self):
1126         d = self.shouldFail2(error.Error,
1127                              "test_POST_upload_no_link_mutable_toobig",
1128                              "413 Request Entity Too Large",
1129                              "SDMF is limited to one segment, and 10001 > 10000",
1130                              self.POST,
1131                              "/uri", t="upload", mutable="true",
1132                              file=("new.txt",
1133                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1134         return d
1135
1136     def test_POST_upload_mutable(self):
1137         # this creates a mutable file
1138         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1139                       file=("new.txt", self.NEWFILE_CONTENTS))
1140         fn = self._foo_node
1141         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1142         d.addCallback(lambda res:
1143                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1144                                                              self.NEWFILE_CONTENTS))
1145         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1146         def _got(newnode):
1147             self.failUnless(IMutableFileNode.providedBy(newnode))
1148             self.failUnless(newnode.is_mutable())
1149             self.failIf(newnode.is_readonly())
1150             self._mutable_node = newnode
1151             self._mutable_uri = newnode.get_uri()
1152         d.addCallback(_got)
1153
1154         # now upload it again and make sure that the URI doesn't change
1155         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1156         d.addCallback(lambda res:
1157                       self.POST(self.public_url + "/foo", t="upload",
1158                                 mutable="true",
1159                                 file=("new.txt", NEWER_CONTENTS)))
1160         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1161         d.addCallback(lambda res:
1162                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1163                                                              NEWER_CONTENTS))
1164         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1165         def _got2(newnode):
1166             self.failUnless(IMutableFileNode.providedBy(newnode))
1167             self.failUnless(newnode.is_mutable())
1168             self.failIf(newnode.is_readonly())
1169             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1170         d.addCallback(_got2)
1171
1172         # upload a second time, using PUT instead of POST
1173         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1174         d.addCallback(lambda res:
1175                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1176         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1177         d.addCallback(lambda res:
1178                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1179                                                              NEW2_CONTENTS))
1180
1181         # finally list the directory, since mutable files are displayed
1182         # differently
1183
1184         d.addCallback(lambda res:
1185                       self.GET(self.public_url + "/foo/",
1186                                followRedirect=True))
1187         def _check_page(res):
1188             # TODO: assert more about the contents
1189             self.failUnless("Overwrite" in res)
1190             self.failUnless("Choose new file:" in res)
1191             return res
1192         d.addCallback(_check_page)
1193
1194         # test that clicking on the "overwrite" button works
1195         EVEN_NEWER_CONTENTS = NEWER_CONTENTS + "even newer\n"
1196         def _parse_overwrite_form_and_submit(res):
1197
1198             OVERWRITE_FORM_RE=re.compile('<form action="([^"]*)" method="post" .*<input type="hidden" name="t" value="upload" /><input type="hidden" name="when_done" value="([^"]*)" />', re.I)
1199             mo = OVERWRITE_FORM_RE.search(res)
1200             self.failUnless(mo, "overwrite form not found in '" + res +
1201                             "', in which the overwrite form was not found")
1202             formaction=mo.group(1)
1203             formwhendone=mo.group(2)
1204
1205             fileurl = "../../../uri/" + urllib.quote(self._mutable_uri)
1206             self.failUnlessEqual(formaction, fileurl)
1207             # to POST, we need to absoluteify the URL
1208             new_formaction = "/uri/%s" % urllib.quote(self._mutable_uri)
1209             self.failUnlessEqual(formwhendone,
1210                                  "../uri/%s/" % urllib.quote(self._foo_uri))
1211             return self.POST(new_formaction,
1212                              t="upload",
1213                              file=("new.txt", EVEN_NEWER_CONTENTS),
1214                              when_done=formwhendone,
1215                              followRedirect=False)
1216         d.addCallback(_parse_overwrite_form_and_submit)
1217         # This will redirect us to ../uri/$FOOURI, rather than
1218         # ../uri/$PARENT/foo, but apparently twisted.web.client absolutifies
1219         # the redirect for us, and remember that shouldRedirect prepends
1220         # self.webish_url for us.
1221         d.addBoth(self.shouldRedirect,
1222                   "/uri/%s/" % urllib.quote(self._foo_uri),
1223                   which="test_POST_upload_mutable.overwrite")
1224         d.addCallback(lambda res:
1225                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1226                                                              EVEN_NEWER_CONTENTS))
1227         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1228         def _got3(newnode):
1229             self.failUnless(IMutableFileNode.providedBy(newnode))
1230             self.failUnless(newnode.is_mutable())
1231             self.failIf(newnode.is_readonly())
1232             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1233         d.addCallback(_got3)
1234
1235         # look at the JSON form of the enclosing directory
1236         d.addCallback(lambda res:
1237                       self.GET(self.public_url + "/foo/?t=json",
1238                                followRedirect=True))
1239         def _check_page_json(res):
1240             parsed = simplejson.loads(res)
1241             self.failUnlessEqual(parsed[0], "dirnode")
1242             children = parsed[1]["children"]
1243             self.failUnless("new.txt" in children)
1244             new_json = children["new.txt"]
1245             self.failUnlessEqual(new_json[0], "filenode")
1246             self.failUnless(new_json[1]["mutable"])
1247             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1248             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1249             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1250         d.addCallback(_check_page_json)
1251
1252         # and the JSON form of the file
1253         d.addCallback(lambda res:
1254                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1255         def _check_file_json(res):
1256             parsed = simplejson.loads(res)
1257             self.failUnlessEqual(parsed[0], "filenode")
1258             self.failUnless(parsed[1]["mutable"])
1259             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1260             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1261             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1262         d.addCallback(_check_file_json)
1263
1264         # and look at t=uri and t=readonly-uri
1265         d.addCallback(lambda res:
1266                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1267         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1268         d.addCallback(lambda res:
1269                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1270         def _check_ro_uri(res):
1271             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1272             self.failUnlessEqual(res, ro_uri)
1273         d.addCallback(_check_ro_uri)
1274
1275         # make sure we can get to it from /uri/URI
1276         d.addCallback(lambda res:
1277                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1278         d.addCallback(lambda res:
1279                       self.failUnlessEqual(res, EVEN_NEWER_CONTENTS))
1280
1281         # and that HEAD computes the size correctly
1282         d.addCallback(lambda res:
1283                       self.HEAD(self.public_url + "/foo/new.txt"))
1284         def _got_headers(headers):
1285             self.failUnlessEqual(headers["content-length"][0],
1286                                  str(len(EVEN_NEWER_CONTENTS)))
1287             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1288         d.addCallback(_got_headers)
1289
1290         # make sure that size errors are displayed correctly for overwrite
1291         d.addCallback(lambda res:
1292                       self.shouldFail2(error.Error,
1293                                        "test_POST_upload_mutable-toobig",
1294                                        "413 Request Entity Too Large",
1295                                        "SDMF is limited to one segment, and 10001 > 10000",
1296                                        self.POST,
1297                                        self.public_url + "/foo", t="upload",
1298                                        mutable="true",
1299                                        file=("new.txt",
1300                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1301                                        ))
1302
1303         d.addErrback(self.dump_error)
1304         return d
1305
1306     def test_POST_upload_mutable_toobig(self):
1307         d = self.shouldFail2(error.Error,
1308                              "test_POST_upload_no_link_mutable_toobig",
1309                              "413 Request Entity Too Large",
1310                              "SDMF is limited to one segment, and 10001 > 10000",
1311                              self.POST,
1312                              self.public_url + "/foo",
1313                              t="upload", mutable="true",
1314                              file=("new.txt",
1315                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1316         return d
1317
1318     def dump_error(self, f):
1319         # if the web server returns an error code (like 400 Bad Request),
1320         # web.client.getPage puts the HTTP response body into the .response
1321         # attribute of the exception object that it gives back. It does not
1322         # appear in the Failure's repr(), so the ERROR that trial displays
1323         # will be rather terse and unhelpful. addErrback this method to the
1324         # end of your chain to get more information out of these errors.
1325         if f.check(error.Error):
1326             print "web.error.Error:"
1327             print f
1328             print f.value.response
1329         return f
1330
1331     def test_POST_upload_replace(self):
1332         d = self.POST(self.public_url + "/foo", t="upload",
1333                       file=("bar.txt", self.NEWFILE_CONTENTS))
1334         fn = self._foo_node
1335         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1336         d.addCallback(lambda res:
1337                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1338                                                       self.NEWFILE_CONTENTS))
1339         return d
1340
1341     def test_POST_upload_no_replace_ok(self):
1342         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1343                       file=("new.txt", self.NEWFILE_CONTENTS))
1344         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1345         d.addCallback(lambda res: self.failUnlessEqual(res,
1346                                                        self.NEWFILE_CONTENTS))
1347         return d
1348
1349     def test_POST_upload_no_replace_queryarg(self):
1350         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1351                       file=("bar.txt", self.NEWFILE_CONTENTS))
1352         d.addBoth(self.shouldFail, error.Error,
1353                   "POST_upload_no_replace_queryarg",
1354                   "409 Conflict",
1355                   "There was already a child by that name, and you asked me "
1356                   "to not replace it")
1357         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1358         d.addCallback(self.failUnlessIsBarDotTxt)
1359         return d
1360
1361     def test_POST_upload_no_replace_field(self):
1362         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1363                       file=("bar.txt", self.NEWFILE_CONTENTS))
1364         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1365                   "409 Conflict",
1366                   "There was already a child by that name, and you asked me "
1367                   "to not replace it")
1368         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1369         d.addCallback(self.failUnlessIsBarDotTxt)
1370         return d
1371
1372     def test_POST_upload_whendone(self):
1373         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1374                       file=("new.txt", self.NEWFILE_CONTENTS))
1375         d.addBoth(self.shouldRedirect, "/THERE")
1376         fn = self._foo_node
1377         d.addCallback(lambda res:
1378                       self.failUnlessChildContentsAre(fn, u"new.txt",
1379                                                       self.NEWFILE_CONTENTS))
1380         return d
1381
1382     def test_POST_upload_named(self):
1383         fn = self._foo_node
1384         d = self.POST(self.public_url + "/foo", t="upload",
1385                       name="new.txt", file=self.NEWFILE_CONTENTS)
1386         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1387         d.addCallback(lambda res:
1388                       self.failUnlessChildContentsAre(fn, u"new.txt",
1389                                                       self.NEWFILE_CONTENTS))
1390         return d
1391
1392     def test_POST_upload_named_badfilename(self):
1393         d = self.POST(self.public_url + "/foo", t="upload",
1394                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1395         d.addBoth(self.shouldFail, error.Error,
1396                   "test_POST_upload_named_badfilename",
1397                   "400 Bad Request",
1398                   "name= may not contain a slash",
1399                   )
1400         # make sure that nothing was added
1401         d.addCallback(lambda res:
1402                       self.failUnlessNodeKeysAre(self._foo_node,
1403                                                  [u"bar.txt", u"blockingfile",
1404                                                   u"empty", u"n\u00fc.txt",
1405                                                   u"sub"]))
1406         return d
1407
1408     def test_POST_FILEURL_check(self):
1409         bar_url = self.public_url + "/foo/bar.txt"
1410         d = self.POST(bar_url, t="check")
1411         def _check(res):
1412             self.failUnless("Healthy!" in res)
1413         d.addCallback(_check)
1414         redir_url = "http://allmydata.org/TARGET"
1415         def _check2(statuscode, target):
1416             self.failUnlessEqual(statuscode, str(http.FOUND))
1417             self.failUnlessEqual(target, redir_url)
1418         d.addCallback(lambda res:
1419                       self.shouldRedirect2("test_POST_FILEURL_check",
1420                                            _check2,
1421                                            self.POST, bar_url,
1422                                            t="check",
1423                                            when_done=redir_url))
1424         d.addCallback(lambda res:
1425                       self.POST(bar_url, t="check", return_to=redir_url))
1426         def _check3(res):
1427             self.failUnless("Healthy!" in res)
1428             self.failUnless("Return to parent directory" in res)
1429             self.failUnless(redir_url in res)
1430         d.addCallback(_check3)
1431         return d
1432
1433     def test_POST_DIRURL_check(self):
1434         foo_url = self.public_url + "/foo/"
1435         d = self.POST(foo_url, t="check")
1436         def _check(res):
1437             self.failUnless("Healthy!" in res)
1438         d.addCallback(_check)
1439         redir_url = "http://allmydata.org/TARGET"
1440         def _check2(statuscode, target):
1441             self.failUnlessEqual(statuscode, str(http.FOUND))
1442             self.failUnlessEqual(target, redir_url)
1443         d.addCallback(lambda res:
1444                       self.shouldRedirect2("test_POST_DIRURL_check",
1445                                            _check2,
1446                                            self.POST, foo_url,
1447                                            t="check",
1448                                            when_done=redir_url))
1449         d.addCallback(lambda res:
1450                       self.POST(foo_url, t="check", return_to=redir_url))
1451         def _check3(res):
1452             self.failUnless("Healthy!" in res)
1453             self.failUnless("Return to parent directory" in res)
1454             self.failUnless(redir_url in res)
1455         d.addCallback(_check3)
1456         return d
1457
1458     def test_POST_DIRURL_deepcheck(self):
1459         d = self.POST(self.public_url, t="deep-check")
1460         def _check(res):
1461             self.failUnless("Objects Checked: <span>8</span>" in res)
1462             self.failUnless("Objects Healthy: <span>8</span>" in res)
1463             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1464             self.failUnless("Repairs Successful: <span>0</span>" in res)
1465         d.addCallback(_check)
1466         redir_url = "http://allmydata.org/TARGET"
1467         def _check2(statuscode, target):
1468             self.failUnlessEqual(statuscode, str(http.FOUND))
1469             self.failUnlessEqual(target, redir_url)
1470         d.addCallback(lambda res:
1471                       self.shouldRedirect2("test_POST_DIRURL_check",
1472                                            _check2,
1473                                            self.POST, self.public_url,
1474                                            t="deep-check",
1475                                            when_done=redir_url))
1476         d.addCallback(lambda res:
1477                       self.POST(self.public_url, t="deep-check",
1478                                 return_to=redir_url))
1479         def _check3(res):
1480             self.failUnless("Return to parent directory" in res)
1481             self.failUnless(redir_url in res)
1482         d.addCallback(_check3)
1483         return d
1484
1485     def test_POST_FILEURL_bad_t(self):
1486         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1487                              "POST to file: bad t=bogus",
1488                              self.POST, self.public_url + "/foo/bar.txt",
1489                              t="bogus")
1490         return d
1491
1492     def test_POST_mkdir(self): # return value?
1493         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1494         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1495         d.addCallback(self.failUnlessNodeKeysAre, [])
1496         return d
1497
1498     def test_POST_mkdir_2(self):
1499         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1500         d.addCallback(lambda res:
1501                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1502         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1503         d.addCallback(self.failUnlessNodeKeysAre, [])
1504         return d
1505
1506     def test_POST_mkdirs_2(self):
1507         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1508         d.addCallback(lambda res:
1509                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1510         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1511         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1512         d.addCallback(self.failUnlessNodeKeysAre, [])
1513         return d
1514
1515     def test_POST_mkdir_no_parentdir_noredirect(self):
1516         d = self.POST("/uri?t=mkdir")
1517         def _after_mkdir(res):
1518             uri.NewDirectoryURI.init_from_string(res)
1519         d.addCallback(_after_mkdir)
1520         return d
1521
1522     def test_POST_mkdir_no_parentdir_redirect(self):
1523         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1524         d.addBoth(self.shouldRedirect, None, statuscode='303')
1525         def _check_target(target):
1526             target = urllib.unquote(target)
1527             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1528         d.addCallback(_check_target)
1529         return d
1530
1531     def test_POST_noparent_bad(self):
1532         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1533                                   "/uri accepts only PUT, PUT?t=mkdir, "
1534                                   "POST?t=upload, and POST?t=mkdir",
1535                                   self.POST, "/uri?t=bogus")
1536         return d
1537
1538     def test_welcome_page_mkdir_button(self):
1539         # Fetch the welcome page.
1540         d = self.GET("/")
1541         def _after_get_welcome_page(res):
1542             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1543             mo = MKDIR_BUTTON_RE.search(res)
1544             formaction = mo.group(1)
1545             formt = mo.group(2)
1546             formaname = mo.group(3)
1547             formavalue = mo.group(4)
1548             return (formaction, formt, formaname, formavalue)
1549         d.addCallback(_after_get_welcome_page)
1550         def _after_parse_form(res):
1551             (formaction, formt, formaname, formavalue) = res
1552             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1553         d.addCallback(_after_parse_form)
1554         d.addBoth(self.shouldRedirect, None, statuscode='303')
1555         return d
1556
1557     def test_POST_mkdir_replace(self): # return value?
1558         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1559         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1560         d.addCallback(self.failUnlessNodeKeysAre, [])
1561         return d
1562
1563     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1564         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1565         d.addBoth(self.shouldFail, error.Error,
1566                   "POST_mkdir_no_replace_queryarg",
1567                   "409 Conflict",
1568                   "There was already a child by that name, and you asked me "
1569                   "to not replace it")
1570         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1571         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1572         return d
1573
1574     def test_POST_mkdir_no_replace_field(self): # return value?
1575         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1576                       replace="false")
1577         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1578                   "409 Conflict",
1579                   "There was already a child by that name, and you asked me "
1580                   "to not replace it")
1581         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1582         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1583         return d
1584
1585     def test_POST_mkdir_whendone_field(self):
1586         d = self.POST(self.public_url + "/foo",
1587                       t="mkdir", name="newdir", when_done="/THERE")
1588         d.addBoth(self.shouldRedirect, "/THERE")
1589         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1590         d.addCallback(self.failUnlessNodeKeysAre, [])
1591         return d
1592
1593     def test_POST_mkdir_whendone_queryarg(self):
1594         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1595                       t="mkdir", name="newdir")
1596         d.addBoth(self.shouldRedirect, "/THERE")
1597         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1598         d.addCallback(self.failUnlessNodeKeysAre, [])
1599         return d
1600
1601     def test_POST_bad_t(self):
1602         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1603                              "POST to a directory with bad t=BOGUS",
1604                              self.POST, self.public_url + "/foo", t="BOGUS")
1605         return d
1606
1607     def test_POST_set_children(self):
1608         contents9, n9, newuri9 = self.makefile(9)
1609         contents10, n10, newuri10 = self.makefile(10)
1610         contents11, n11, newuri11 = self.makefile(11)
1611
1612         reqbody = """{
1613                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1614                                                 "size": 0,
1615                                                 "metadata": {
1616                                                   "ctime": 1002777696.7564139,
1617                                                   "mtime": 1002777696.7564139
1618                                                  }
1619                                                } ],
1620                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1621                                                 "size": 1,
1622                                                 "metadata": {
1623                                                   "ctime": 1002777696.7564139,
1624                                                   "mtime": 1002777696.7564139
1625                                                  }
1626                                                } ],
1627                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1628                                                 "size": 2,
1629                                                 "metadata": {
1630                                                   "ctime": 1002777696.7564139,
1631                                                   "mtime": 1002777696.7564139
1632                                                  }
1633                                                } ]
1634                     }""" % (newuri9, newuri10, newuri11)
1635
1636         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1637
1638         d = client.getPage(url, method="POST", postdata=reqbody)
1639         def _then(res):
1640             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1641             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1642             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1643
1644         d.addCallback(_then)
1645         d.addErrback(self.dump_error)
1646         return d
1647
1648     def test_POST_put_uri(self):
1649         contents, n, newuri = self.makefile(8)
1650         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1651         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1652         d.addCallback(lambda res:
1653                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1654                                                       contents))
1655         return d
1656
1657     def test_POST_put_uri_replace(self):
1658         contents, n, newuri = self.makefile(8)
1659         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1660         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1661         d.addCallback(lambda res:
1662                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1663                                                       contents))
1664         return d
1665
1666     def test_POST_put_uri_no_replace_queryarg(self):
1667         contents, n, newuri = self.makefile(8)
1668         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1669                       name="bar.txt", uri=newuri)
1670         d.addBoth(self.shouldFail, error.Error,
1671                   "POST_put_uri_no_replace_queryarg",
1672                   "409 Conflict",
1673                   "There was already a child by that name, and you asked me "
1674                   "to not replace it")
1675         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1676         d.addCallback(self.failUnlessIsBarDotTxt)
1677         return d
1678
1679     def test_POST_put_uri_no_replace_field(self):
1680         contents, n, newuri = self.makefile(8)
1681         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1682                       name="bar.txt", uri=newuri)
1683         d.addBoth(self.shouldFail, error.Error,
1684                   "POST_put_uri_no_replace_field",
1685                   "409 Conflict",
1686                   "There was already a child by that name, and you asked me "
1687                   "to not replace it")
1688         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1689         d.addCallback(self.failUnlessIsBarDotTxt)
1690         return d
1691
1692     def test_POST_delete(self):
1693         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1694         d.addCallback(lambda res: self._foo_node.list())
1695         def _check(children):
1696             self.failIf(u"bar.txt" in children)
1697         d.addCallback(_check)
1698         return d
1699
1700     def test_POST_rename_file(self):
1701         d = self.POST(self.public_url + "/foo", t="rename",
1702                       from_name="bar.txt", to_name='wibble.txt')
1703         d.addCallback(lambda res:
1704                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1705         d.addCallback(lambda res:
1706                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1707         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1708         d.addCallback(self.failUnlessIsBarDotTxt)
1709         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1710         d.addCallback(self.failUnlessIsBarJSON)
1711         return d
1712
1713     def test_POST_rename_file_replace(self):
1714         # rename a file and replace a directory with it
1715         d = self.POST(self.public_url + "/foo", t="rename",
1716                       from_name="bar.txt", to_name='empty')
1717         d.addCallback(lambda res:
1718                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1719         d.addCallback(lambda res:
1720                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
1721         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
1722         d.addCallback(self.failUnlessIsBarDotTxt)
1723         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1724         d.addCallback(self.failUnlessIsBarJSON)
1725         return d
1726
1727     def test_POST_rename_file_no_replace_queryarg(self):
1728         # rename a file and replace a directory with it
1729         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
1730                       from_name="bar.txt", to_name='empty')
1731         d.addBoth(self.shouldFail, error.Error,
1732                   "POST_rename_file_no_replace_queryarg",
1733                   "409 Conflict",
1734                   "There was already a child by that name, and you asked me "
1735                   "to not replace it")
1736         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1737         d.addCallback(self.failUnlessIsEmptyJSON)
1738         return d
1739
1740     def test_POST_rename_file_no_replace_field(self):
1741         # rename a file and replace a directory with it
1742         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
1743                       from_name="bar.txt", to_name='empty')
1744         d.addBoth(self.shouldFail, error.Error,
1745                   "POST_rename_file_no_replace_field",
1746                   "409 Conflict",
1747                   "There was already a child by that name, and you asked me "
1748                   "to not replace it")
1749         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1750         d.addCallback(self.failUnlessIsEmptyJSON)
1751         return d
1752
1753     def failUnlessIsEmptyJSON(self, res):
1754         data = simplejson.loads(res)
1755         self.failUnlessEqual(data[0], "dirnode", data)
1756         self.failUnlessEqual(len(data[1]["children"]), 0)
1757
1758     def test_POST_rename_file_slash_fail(self):
1759         d = self.POST(self.public_url + "/foo", t="rename",
1760                       from_name="bar.txt", to_name='kirk/spock.txt')
1761         d.addBoth(self.shouldFail, error.Error,
1762                   "test_POST_rename_file_slash_fail",
1763                   "400 Bad Request",
1764                   "to_name= may not contain a slash",
1765                   )
1766         d.addCallback(lambda res:
1767                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1768         d.addCallback(lambda res: self.POST(self.public_url, t="rename",
1769                       from_name="foo/bar.txt", to_name='george.txt'))
1770         d.addBoth(self.shouldFail, error.Error,
1771                   "test_POST_rename_file_slash_fail",
1772                   "400 Bad Request",
1773                   "from_name= may not contain a slash",
1774                   )
1775         d.addCallback(lambda res:
1776                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1777         d.addCallback(lambda res:
1778                       self.failIfNodeHasChild(self.public_root, u"george.txt"))
1779         d.addCallback(lambda res:
1780                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1781         d.addCallback(lambda res: self.GET(self.public_url + "/foo?t=json"))
1782         d.addCallback(self.failUnlessIsFooJSON)
1783         return d
1784
1785     def test_POST_rename_dir(self):
1786         d = self.POST(self.public_url, t="rename",
1787                       from_name="foo", to_name='plunk')
1788         d.addCallback(lambda res:
1789                       self.failIfNodeHasChild(self.public_root, u"foo"))
1790         d.addCallback(lambda res:
1791                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
1792         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
1793         d.addCallback(self.failUnlessIsFooJSON)
1794         return d
1795
1796     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
1797         """ If target is not None then the redirection has to go to target.  If
1798         statuscode is not None then the redirection has to be accomplished with
1799         that HTTP status code."""
1800         if not isinstance(res, failure.Failure):
1801             to_where = (target is None) and "somewhere" or ("to " + target)
1802             self.fail("%s: we were expecting to get redirected %s, not get an"
1803                       " actual page: %s" % (which, to_where, res))
1804         res.trap(error.PageRedirect)
1805         if statuscode is not None:
1806             self.failUnlessEqual(res.value.status, statuscode,
1807                                  "%s: not a redirect" % which)
1808         if target is not None:
1809             # the PageRedirect does not seem to capture the uri= query arg
1810             # properly, so we can't check for it.
1811             realtarget = self.webish_url + target
1812             self.failUnlessEqual(res.value.location, realtarget,
1813                                  "%s: wrong target" % which)
1814         return res.value.location
1815
1816     def test_GET_URI_form(self):
1817         base = "/uri?uri=%s" % self._bar_txt_uri
1818         # this is supposed to give us a redirect to /uri/$URI, plus arguments
1819         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1820         d = self.GET(base)
1821         d.addBoth(self.shouldRedirect, targetbase)
1822         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
1823         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
1824         d.addCallback(lambda res: self.GET(base+"&t=json"))
1825         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
1826         d.addCallback(self.log, "about to get file by uri")
1827         d.addCallback(lambda res: self.GET(base, followRedirect=True))
1828         d.addCallback(self.failUnlessIsBarDotTxt)
1829         d.addCallback(self.log, "got file by uri, about to get dir by uri")
1830         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
1831                                            followRedirect=True))
1832         d.addCallback(self.failUnlessIsFooJSON)
1833         d.addCallback(self.log, "got dir by uri")
1834
1835         return d
1836
1837     def test_GET_URI_form_bad(self):
1838         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
1839                              "400 Bad Request", "GET /uri requires uri=",
1840                              self.GET, "/uri")
1841         return d
1842
1843     def test_GET_rename_form(self):
1844         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
1845                      followRedirect=True)
1846         def _check(res):
1847             self.failUnless('name="when_done" value="."' in res, res)
1848             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
1849         d.addCallback(_check)
1850         return d
1851
1852     def log(self, res, msg):
1853         #print "MSG: %s  RES: %s" % (msg, res)
1854         log.msg(msg)
1855         return res
1856
1857     def test_GET_URI_URL(self):
1858         base = "/uri/%s" % self._bar_txt_uri
1859         d = self.GET(base)
1860         d.addCallback(self.failUnlessIsBarDotTxt)
1861         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
1862         d.addCallback(self.failUnlessIsBarDotTxt)
1863         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
1864         d.addCallback(self.failUnlessIsBarDotTxt)
1865         return d
1866
1867     def test_GET_URI_URL_dir(self):
1868         base = "/uri/%s?t=json" % self._foo_uri
1869         d = self.GET(base)
1870         d.addCallback(self.failUnlessIsFooJSON)
1871         return d
1872
1873     def test_GET_URI_URL_missing(self):
1874         base = "/uri/%s" % self._bad_file_uri
1875         d = self.GET(base)
1876         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
1877                   http.GONE, response_substring="NotEnoughSharesError")
1878         # TODO: how can we exercise both sides of WebDownloadTarget.fail
1879         # here? we must arrange for a download to fail after target.open()
1880         # has been called, and then inspect the response to see that it is
1881         # shorter than we expected.
1882         return d
1883
1884     def test_PUT_NEWFILEURL_uri(self):
1885         contents, n, new_uri = self.makefile(8)
1886         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
1887         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1888         d.addCallback(lambda res:
1889                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1890                                                       contents))
1891         return d
1892
1893     def test_PUT_NEWFILEURL_uri_replace(self):
1894         contents, n, new_uri = self.makefile(8)
1895         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
1896         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1897         d.addCallback(lambda res:
1898                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1899                                                       contents))
1900         return d
1901
1902     def test_PUT_NEWFILEURL_uri_no_replace(self):
1903         contents, n, new_uri = self.makefile(8)
1904         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
1905         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
1906                   "409 Conflict",
1907                   "There was already a child by that name, and you asked me "
1908                   "to not replace it")
1909         return d
1910
1911     def test_PUT_NEWFILE_URI(self):
1912         file_contents = "New file contents here\n"
1913         d = self.PUT("/uri", file_contents)
1914         def _check(uri):
1915             self.failUnless(uri in FakeCHKFileNode.all_contents)
1916             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
1917                                  file_contents)
1918             return self.GET("/uri/%s" % uri)
1919         d.addCallback(_check)
1920         def _check2(res):
1921             self.failUnlessEqual(res, file_contents)
1922         d.addCallback(_check2)
1923         return d
1924
1925     def test_PUT_NEWFILE_URI_only_PUT(self):
1926         d = self.PUT("/uri?t=bogus", "")
1927         d.addBoth(self.shouldFail, error.Error,
1928                   "PUT_NEWFILE_URI_only_PUT",
1929                   "400 Bad Request",
1930                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
1931         return d
1932
1933     def test_PUT_NEWFILE_URI_mutable(self):
1934         file_contents = "New file contents here\n"
1935         d = self.PUT("/uri?mutable=true", file_contents)
1936         def _check_mutable(uri):
1937             uri = uri.strip()
1938             u = IURI(uri)
1939             self.failUnless(IMutableFileURI.providedBy(u))
1940             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1941             n = self.s.create_node_from_uri(uri)
1942             return n.download_best_version()
1943         d.addCallback(_check_mutable)
1944         def _check2_mutable(data):
1945             self.failUnlessEqual(data, file_contents)
1946         d.addCallback(_check2_mutable)
1947         return d
1948
1949         def _check(uri):
1950             self.failUnless(uri in FakeCHKFileNode.all_contents)
1951             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
1952                                  file_contents)
1953             return self.GET("/uri/%s" % uri)
1954         d.addCallback(_check)
1955         def _check2(res):
1956             self.failUnlessEqual(res, file_contents)
1957         d.addCallback(_check2)
1958         return d
1959
1960     def test_PUT_mkdir(self):
1961         d = self.PUT("/uri?t=mkdir", "")
1962         def _check(uri):
1963             n = self.s.create_node_from_uri(uri.strip())
1964             d2 = self.failUnlessNodeKeysAre(n, [])
1965             d2.addCallback(lambda res:
1966                            self.GET("/uri/%s?t=json" % uri))
1967             return d2
1968         d.addCallback(_check)
1969         d.addCallback(self.failUnlessIsEmptyJSON)
1970         return d
1971
1972     def test_POST_check(self):
1973         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
1974         def _done(res):
1975             # this returns a string form of the results, which are probably
1976             # None since we're using fake filenodes.
1977             # TODO: verify that the check actually happened, by changing
1978             # FakeCHKFileNode to count how many times .check() has been
1979             # called.
1980             pass
1981         d.addCallback(_done)
1982         return d
1983
1984     def test_bad_method(self):
1985         url = self.webish_url + self.public_url + "/foo/bar.txt"
1986         d = self.shouldHTTPError2("test_bad_method",
1987                                   501, "Not Implemented",
1988                                   "I don't know how to treat a BOGUS request.",
1989                                   client.getPage, url, method="BOGUS")
1990         return d
1991
1992     def test_short_url(self):
1993         url = self.webish_url + "/uri"
1994         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
1995                                   "I don't know how to treat a DELETE request.",
1996                                   client.getPage, url, method="DELETE")
1997         return d
1998
1999 class Util(unittest.TestCase):
2000     def test_abbreviate_time(self):
2001         self.failUnlessEqual(common.abbreviate_time(None), "")
2002         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2003         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2004         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2005         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2006
2007     def test_abbreviate_rate(self):
2008         self.failUnlessEqual(common.abbreviate_rate(None), "")
2009         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2010         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2011         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2012
2013     def test_abbreviate_size(self):
2014         self.failUnlessEqual(common.abbreviate_size(None), "")
2015         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2016         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2017         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2018         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2019