src/allmydata/test/test_web.py

   1 import re, urllib
   2 import simplejson
   3 from twisted.application import service
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, reactor
   6 from twisted.web import client, error, http
   7 from twisted.python import failure, log
   8 from allmydata import interfaces, provisioning, uri, webish
   9 from allmydata.immutable import upload, download
  10 from allmydata.web import status, common
  11 from allmydata.util import fileutil
  12 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  13      FakeMutableFileNode, create_chk_filenode
  14 from allmydata.interfaces import IURI, INewDirectoryURI, \
  15      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  16 from allmydata.mutable import servermap, publish, retrieve
  17
  18 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  19 # create a webserver that works against them
  20
  21 class FakeIntroducerClient:
  22     def get_all_connectors(self):
  23         return {}
  24     def get_all_connections_for(self, service_name):
  25         return frozenset()
  26     def get_all_peerids(self):
  27         return frozenset()
  28
  29 class FakeClient(service.MultiService):
  30     nodeid = "fake_nodeid"
  31     nickname = "fake_nickname"
  32     basedir = "fake_basedir"
  33     def get_versions(self):
  34         return {'allmydata': "fake",
  35                 'foolscap': "fake",
  36                 'twisted': "fake",
  37                 'zfec': "fake",
  38                 }
  39     introducer_furl = "None"
  40     introducer_client = FakeIntroducerClient()
  41     _all_upload_status = [upload.UploadStatus()]
  42     _all_download_status = [download.DownloadStatus()]
  43     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  44     _all_publish_statuses = [publish.PublishStatus()]
  45     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  46     convergence = "some random string"
  47
  48     def connected_to_introducer(self):
  49         return False
  50
  51     def create_node_from_uri(self, auri):
  52         u = uri.from_string(auri)
  53         if (INewDirectoryURI.providedBy(u)
  54             or IReadonlyNewDirectoryURI.providedBy(u)):
  55             return FakeDirectoryNode(self).init_from_uri(u)
  56         if IFileURI.providedBy(u):
  57             return FakeCHKFileNode(u, self)
  58         assert IMutableFileURI.providedBy(u), u
  59         return FakeMutableFileNode(self).init_from_uri(u)
  60
  61     def create_empty_dirnode(self):
  62         n = FakeDirectoryNode(self)
  63         d = n.create()
  64         d.addCallback(lambda res: n)
  65         return d
  66
  67     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  68     def create_mutable_file(self, contents=""):
  69         n = FakeMutableFileNode(self)
  70         return n.create(contents)
  71
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(self, data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83
  84     def list_all_upload_statuses(self):
  85         return self._all_upload_status
  86     def list_all_download_statuses(self):
  87         return self._all_download_status
  88     def list_all_mapupdate_statuses(self):
  89         return self._all_mapupdate_statuses
  90     def list_all_publish_statuses(self):
  91         return self._all_publish_statuses
  92     def list_all_retrieve_statuses(self):
  93         return self._all_retrieve_statuses
  94     def list_all_helper_statuses(self):
  95         return []
  96
  97 class HTTPClientHEADFactory(client.HTTPClientFactory):
  98     def __init__(self, *args, **kwargs):
  99         client.HTTPClientFactory.__init__(self, *args, **kwargs)
 100         self.deferred.addCallback(lambda res: self.response_headers)
 101
 102
 103 class WebMixin(object):
 104     def setUp(self):
 105         self.s = FakeClient()
 106         self.s.startService()
 107         self.ws = s = webish.WebishServer("0")
 108         s.setServiceParent(self.s)
 109         self.webish_port = port = s.listener._port.getHost().port
 110         self.webish_url = "http://localhost:%d" % port
 111
 112         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 113         d = defer.DeferredList(l)
 114         def _then(res):
 115             self.public_root = res[0][1]
 116             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 117             self.public_url = "/uri/" + self.public_root.get_uri()
 118             self.private_root = res[1][1]
 119
 120             foo = res[2][1]
 121             self._foo_node = foo
 122             self._foo_uri = foo.get_uri()
 123             self._foo_readonly_uri = foo.get_readonly_uri()
 124             # NOTE: we ignore the deferred on all set_uri() calls, because we
 125             # know the fake nodes do these synchronously
 126             self.public_root.set_uri(u"foo", foo.get_uri())
 127
 128             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 129             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 130
 131             foo.set_uri(u"empty", res[3][1].get_uri())
 132             sub_uri = res[4][1].get_uri()
 133             self._sub_uri = sub_uri
 134             foo.set_uri(u"sub", sub_uri)
 135             sub = self.s.create_node_from_uri(sub_uri)
 136
 137             _ign, n, blocking_uri = self.makefile(1)
 138             foo.set_uri(u"blockingfile", blocking_uri)
 139
 140             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 141             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 142             # still think of it as an umlaut
 143             foo.set_uri(unicode_filename, self._bar_txt_uri)
 144
 145             _ign, n, baz_file = self.makefile(2)
 146             sub.set_uri(u"baz.txt", baz_file)
 147
 148             _ign, n, self._bad_file_uri = self.makefile(3)
 149             # this uri should not be downloadable
 150             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 151
 152             rodir = res[5][1]
 153             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 154             rodir.set_uri(u"nor", baz_file)
 155
 156             # public/
 157             # public/foo/
 158             # public/foo/bar.txt
 159             # public/foo/blockingfile
 160             # public/foo/empty/
 161             # public/foo/sub/
 162             # public/foo/sub/baz.txt
 163             # public/reedownlee/
 164             # public/reedownlee/nor
 165             self.NEWFILE_CONTENTS = "newfile contents\n"
 166
 167             return foo.get_metadata_for(u"bar.txt")
 168         d.addCallback(_then)
 169         def _got_metadata(metadata):
 170             self._bar_txt_metadata = metadata
 171         d.addCallback(_got_metadata)
 172         return d
 173
 174     def makefile(self, number):
 175         contents = "contents of file %s\n" % number
 176         n = create_chk_filenode(self.s, contents)
 177         return contents, n, n.get_uri()
 178
 179     def tearDown(self):
 180         return self.s.stopService()
 181
 182     def failUnlessIsBarDotTxt(self, res):
 183         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 184
 185     def failUnlessIsBarJSON(self, res):
 186         data = simplejson.loads(res)
 187         self.failUnless(isinstance(data, list))
 188         self.failUnlessEqual(data[0], "filenode")
 189         self.failUnless(isinstance(data[1], dict))
 190         self.failIf(data[1]["mutable"])
 191         self.failIf("rw_uri" in data[1]) # immutable
 192         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 193         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 194
 195     def failUnlessIsFooJSON(self, res):
 196         data = simplejson.loads(res)
 197         self.failUnless(isinstance(data, list))
 198         self.failUnlessEqual(data[0], "dirnode", res)
 199         self.failUnless(isinstance(data[1], dict))
 200         self.failUnless(data[1]["mutable"])
 201         self.failUnless("rw_uri" in data[1]) # mutable
 202         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 203         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 204
 205         kidnames = sorted(data[1]["children"])
 206         self.failUnlessEqual(kidnames,
 207                              [u"bar.txt", u"blockingfile", u"empty",
 208                               u"n\u00fc.txt", u"sub"])
 209         kids = data[1]["children"]
 210         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 211         self.failUnless("metadata" in kids[u"sub"][1])
 212         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 213         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 214         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 215         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 216         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 217         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 218                              self._bar_txt_metadata["ctime"])
 219         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 220                              self._bar_txt_uri)
 221
 222     def GET(self, urlpath, followRedirect=False):
 223         assert not isinstance(urlpath, unicode)
 224         url = self.webish_url + urlpath
 225         return client.getPage(url, method="GET", followRedirect=followRedirect)
 226
 227     def HEAD(self, urlpath):
 228         # this requires some surgery, because twisted.web.client doesn't want
 229         # to give us back the response headers.
 230         factory = HTTPClientHEADFactory(urlpath)
 231         reactor.connectTCP("localhost", self.webish_port, factory)
 232         return factory.deferred
 233
 234     def PUT(self, urlpath, data):
 235         url = self.webish_url + urlpath
 236         return client.getPage(url, method="PUT", postdata=data)
 237
 238     def DELETE(self, urlpath):
 239         url = self.webish_url + urlpath
 240         return client.getPage(url, method="DELETE")
 241
 242     def POST(self, urlpath, followRedirect=False, **fields):
 243         url = self.webish_url + urlpath
 244         sepbase = "boogabooga"
 245         sep = "--" + sepbase
 246         form = []
 247         form.append(sep)
 248         form.append('Content-Disposition: form-data; name="_charset"')
 249         form.append('')
 250         form.append('UTF-8')
 251         form.append(sep)
 252         for name, value in fields.iteritems():
 253             if isinstance(value, tuple):
 254                 filename, value = value
 255                 form.append('Content-Disposition: form-data; name="%s"; '
 256                             'filename="%s"' % (name, filename.encode("utf-8")))
 257             else:
 258                 form.append('Content-Disposition: form-data; name="%s"' % name)
 259             form.append('')
 260             if isinstance(value, unicode):
 261                 value = value.encode("utf-8")
 262             else:
 263                 value = str(value)
 264             assert isinstance(value, str)
 265             form.append(value)
 266             form.append(sep)
 267         form[-1] += "--"
 268         body = "\r\n".join(form) + "\r\n"
 269         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 270                    }
 271         return client.getPage(url, method="POST", postdata=body,
 272                               headers=headers, followRedirect=followRedirect)
 273
 274     def shouldFail(self, res, expected_failure, which,
 275                    substring=None, response_substring=None):
 276         if isinstance(res, failure.Failure):
 277             res.trap(expected_failure)
 278             if substring:
 279                 self.failUnless(substring in str(res),
 280                                 "substring '%s' not in '%s'"
 281                                 % (substring, str(res)))
 282             if response_substring:
 283                 self.failUnless(response_substring in res.value.response,
 284                                 "response substring '%s' not in '%s'"
 285                                 % (response_substring, res.value.response))
 286         else:
 287             self.fail("%s was supposed to raise %s, not get '%s'" %
 288                       (which, expected_failure, res))
 289
 290     def shouldFail2(self, expected_failure, which, substring,
 291                     response_substring,
 292                     callable, *args, **kwargs):
 293         assert substring is None or isinstance(substring, str)
 294         assert response_substring is None or isinstance(response_substring, str)
 295         d = defer.maybeDeferred(callable, *args, **kwargs)
 296         def done(res):
 297             if isinstance(res, failure.Failure):
 298                 res.trap(expected_failure)
 299                 if substring:
 300                     self.failUnless(substring in str(res),
 301                                     "%s: substring '%s' not in '%s'"
 302                                     % (which, substring, str(res)))
 303                 if response_substring:
 304                     self.failUnless(response_substring in res.value.response,
 305                                     "%s: response substring '%s' not in '%s'"
 306                                     % (which,
 307                                        response_substring, res.value.response))
 308             else:
 309                 self.fail("%s was supposed to raise %s, not get '%s'" %
 310                           (which, expected_failure, res))
 311         d.addBoth(done)
 312         return d
 313
 314     def should404(self, res, which):
 315         if isinstance(res, failure.Failure):
 316             res.trap(error.Error)
 317             self.failUnlessEqual(res.value.status, "404")
 318         else:
 319             self.fail("%s was supposed to Error(404), not get '%s'" %
 320                       (which, res))
 321
 322     def shouldHTTPError(self, res, which, code=None, substring=None,
 323                         response_substring=None):
 324         if isinstance(res, failure.Failure):
 325             res.trap(error.Error)
 326             if code is not None:
 327                 self.failUnlessEqual(res.value.status, str(code))
 328             if substring:
 329                 self.failUnless(substring in str(res),
 330                                 "substring '%s' not in '%s'"
 331                                 % (substring, str(res)))
 332             if response_substring:
 333                 self.failUnless(response_substring in res.value.response,
 334                                 "response substring '%s' not in '%s'"
 335                                 % (response_substring, res.value.response))
 336         else:
 337             self.fail("%s was supposed to Error(%s), not get '%s'" %
 338                       (which, code, res))
 339
 340     def shouldHTTPError2(self, which,
 341                          code=None, substring=None, response_substring=None,
 342                          callable=None, *args, **kwargs):
 343         assert substring is None or isinstance(substring, str)
 344         assert callable
 345         d = defer.maybeDeferred(callable, *args, **kwargs)
 346         d.addBoth(self.shouldHTTPError, which,
 347                   code, substring, response_substring)
 348         return d
 349
 350
 351 class Web(WebMixin, unittest.TestCase):
 352     def test_create(self):
 353         pass
 354
 355     def test_welcome(self):
 356         d = self.GET("/")
 357         def _check(res):
 358             self.failUnless('Welcome To AllMyData' in res)
 359             self.failUnless('Tahoe' in res)
 360
 361             self.s.basedir = 'web/test_welcome'
 362             fileutil.make_dirs("web/test_welcome")
 363             fileutil.make_dirs("web/test_welcome/private")
 364             return self.GET("/")
 365         d.addCallback(_check)
 366         return d
 367
 368     def test_provisioning_math(self):
 369         self.failUnlessEqual(provisioning.binomial(10, 0), 1)
 370         self.failUnlessEqual(provisioning.binomial(10, 1), 10)
 371         self.failUnlessEqual(provisioning.binomial(10, 2), 45)
 372         self.failUnlessEqual(provisioning.binomial(10, 9), 10)
 373         self.failUnlessEqual(provisioning.binomial(10, 10), 1)
 374
 375     def test_provisioning(self):
 376         d = self.GET("/provisioning/")
 377         def _check(res):
 378             self.failUnless('Tahoe Provisioning Tool' in res)
 379             fields = {'filled': True,
 380                       "num_users": int(50e3),
 381                       "files_per_user": 1000,
 382                       "space_per_user": int(1e9),
 383                       "sharing_ratio": 1.0,
 384                       "encoding_parameters": "3-of-10-5",
 385                       "num_servers": 30,
 386                       "ownership_mode": "A",
 387                       "download_rate": 100,
 388                       "upload_rate": 10,
 389                       "delete_rate": 10,
 390                       "lease_timer": 7,
 391                       }
 392             return self.POST("/provisioning/", **fields)
 393
 394         d.addCallback(_check)
 395         def _check2(res):
 396             self.failUnless('Tahoe Provisioning Tool' in res)
 397             self.failUnless("Share space consumed: 167.01TB" in res)
 398
 399             fields = {'filled': True,
 400                       "num_users": int(50e6),
 401                       "files_per_user": 1000,
 402                       "space_per_user": int(5e9),
 403                       "sharing_ratio": 1.0,
 404                       "encoding_parameters": "25-of-100-50",
 405                       "num_servers": 30000,
 406                       "ownership_mode": "E",
 407                       "drive_failure_model": "U",
 408                       "drive_size": 1000,
 409                       "download_rate": 1000,
 410                       "upload_rate": 100,
 411                       "delete_rate": 100,
 412                       "lease_timer": 7,
 413                       }
 414             return self.POST("/provisioning/", **fields)
 415         d.addCallback(_check2)
 416         def _check3(res):
 417             self.failUnless("Share space consumed: huge!" in res)
 418             fields = {'filled': True}
 419             return self.POST("/provisioning/", **fields)
 420         d.addCallback(_check3)
 421         def _check4(res):
 422             self.failUnless("Share space consumed:" in res)
 423         d.addCallback(_check4)
 424         return d
 425
 426     def test_status(self):
 427         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 428         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 429         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 430         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 431         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 432         d = self.GET("/status", followRedirect=True)
 433         def _check(res):
 434             self.failUnless('Upload and Download Status' in res, res)
 435             self.failUnless('"down-%d"' % dl_num in res, res)
 436             self.failUnless('"up-%d"' % ul_num in res, res)
 437             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 438             self.failUnless('"publish-%d"' % pub_num in res, res)
 439             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 440         d.addCallback(_check)
 441         d.addCallback(lambda res: self.GET("/status/?t=json"))
 442         def _check_json(res):
 443             data = simplejson.loads(res)
 444             self.failUnless(isinstance(data, dict))
 445             active = data["active"]
 446             # TODO: test more. We need a way to fake an active operation
 447             # here.
 448         d.addCallback(_check_json)
 449
 450         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 451         def _check_dl(res):
 452             self.failUnless("File Download Status" in res, res)
 453         d.addCallback(_check_dl)
 454         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 455         def _check_ul(res):
 456             self.failUnless("File Upload Status" in res, res)
 457         d.addCallback(_check_ul)
 458         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 459         def _check_mapupdate(res):
 460             self.failUnless("Mutable File Servermap Update Status" in res, res)
 461         d.addCallback(_check_mapupdate)
 462         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 463         def _check_publish(res):
 464             self.failUnless("Mutable File Publish Status" in res, res)
 465         d.addCallback(_check_publish)
 466         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 467         def _check_retrieve(res):
 468             self.failUnless("Mutable File Retrieve Status" in res, res)
 469         d.addCallback(_check_retrieve)
 470
 471         return d
 472
 473     def test_status_numbers(self):
 474         drrm = status.DownloadResultsRendererMixin()
 475         self.failUnlessEqual(drrm.render_time(None, None), "")
 476         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 477         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 478         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 479         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 480         self.failUnlessEqual(drrm.render_rate(None, None), "")
 481         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 482         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 483         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 484
 485         urrm = status.UploadResultsRendererMixin()
 486         self.failUnlessEqual(urrm.render_time(None, None), "")
 487         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 488         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 489         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 490         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 491         self.failUnlessEqual(urrm.render_rate(None, None), "")
 492         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 493         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 494         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 495
 496     def test_GET_FILEURL(self):
 497         d = self.GET(self.public_url + "/foo/bar.txt")
 498         d.addCallback(self.failUnlessIsBarDotTxt)
 499         return d
 500
 501     def test_HEAD_FILEURL(self):
 502         d = self.HEAD(self.public_url + "/foo/bar.txt")
 503         def _got(headers):
 504             self.failUnlessEqual(headers["content-length"][0],
 505                                  str(len(self.BAR_CONTENTS)))
 506             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 507         d.addCallback(_got)
 508         return d
 509
 510     def test_GET_FILEURL_named(self):
 511         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 512         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 513         d = self.GET(base + "/@@name=/blah.txt")
 514         d.addCallback(self.failUnlessIsBarDotTxt)
 515         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 516         d.addCallback(self.failUnlessIsBarDotTxt)
 517         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 518         d.addCallback(self.failUnlessIsBarDotTxt)
 519         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 520         d.addCallback(self.failUnlessIsBarDotTxt)
 521         save_url = base + "?save=true&filename=blah.txt"
 522         d.addCallback(lambda res: self.GET(save_url))
 523         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 524         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 525         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 526         u_url = base + "?save=true&filename=" + u_fn_e
 527         d.addCallback(lambda res: self.GET(u_url))
 528         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 529         return d
 530
 531     def test_PUT_FILEURL_named_bad(self):
 532         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 533         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 534                              "400 Bad Request",
 535                              "/file can only be used with GET or HEAD",
 536                              self.PUT, base + "/@@name=/blah.txt", "")
 537         return d
 538
 539     def test_GET_DIRURL_named_bad(self):
 540         base = "/file/%s" % urllib.quote(self._foo_uri)
 541         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 542                              "400 Bad Request",
 543                              "is not a file-cap",
 544                              self.GET, base + "/@@name=/blah.txt")
 545         return d
 546
 547     def test_GET_slash_file_bad(self):
 548         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 549                              "404 Not Found",
 550                              "/file must be followed by a file-cap and a name",
 551                              self.GET, "/file")
 552         return d
 553
 554     def test_GET_unhandled_URI_named(self):
 555         contents, n, newuri = self.makefile(12)
 556         verifier_cap = n.get_verifier().to_string()
 557         base = "/file/%s" % urllib.quote(verifier_cap)
 558         # client.create_node_from_uri() can't handle verify-caps
 559         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 560                              "400 Bad Request",
 561                              "is not a valid file- or directory- cap",
 562                              self.GET, base)
 563         return d
 564
 565     def test_GET_unhandled_URI(self):
 566         contents, n, newuri = self.makefile(12)
 567         verifier_cap = n.get_verifier().to_string()
 568         base = "/uri/%s" % urllib.quote(verifier_cap)
 569         # client.create_node_from_uri() can't handle verify-caps
 570         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 571                              "400 Bad Request",
 572                              "is not a valid file- or directory- cap",
 573                              self.GET, base)
 574         return d
 575
 576     def test_GET_FILE_URI(self):
 577         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 578         d = self.GET(base)
 579         d.addCallback(self.failUnlessIsBarDotTxt)
 580         return d
 581
 582     def test_GET_FILE_URI_badchild(self):
 583         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 584         errmsg = "Files have no children, certainly not named 'boguschild'"
 585         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 586                              "400 Bad Request", errmsg,
 587                              self.GET, base)
 588         return d
 589
 590     def test_PUT_FILE_URI_badchild(self):
 591         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 592         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 593         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 594                              "400 Bad Request", errmsg,
 595                              self.PUT, base, "")
 596         return d
 597
 598     def test_GET_FILEURL_save(self):
 599         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 600         # TODO: look at the headers, expect a Content-Disposition: attachment
 601         # header.
 602         d.addCallback(self.failUnlessIsBarDotTxt)
 603         return d
 604
 605     def test_GET_FILEURL_missing(self):
 606         d = self.GET(self.public_url + "/foo/missing")
 607         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 608         return d
 609
 610     def test_PUT_NEWFILEURL(self):
 611         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 612         # TODO: we lose the response code, so we can't check this
 613         #self.failUnlessEqual(responsecode, 201)
 614         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 615         d.addCallback(lambda res:
 616                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 617                                                       self.NEWFILE_CONTENTS))
 618         return d
 619
 620     def test_PUT_NEWFILEURL_mutable(self):
 621         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 622                      self.NEWFILE_CONTENTS)
 623         # TODO: we lose the response code, so we can't check this
 624         #self.failUnlessEqual(responsecode, 201)
 625         def _check_uri(res):
 626             u = uri.from_string_mutable_filenode(res)
 627             self.failUnless(u.is_mutable())
 628             self.failIf(u.is_readonly())
 629             return res
 630         d.addCallback(_check_uri)
 631         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 632         d.addCallback(lambda res:
 633                       self.failUnlessMutableChildContentsAre(self._foo_node,
 634                                                              u"new.txt",
 635                                                              self.NEWFILE_CONTENTS))
 636         return d
 637
 638     def test_PUT_NEWFILEURL_mutable_toobig(self):
 639         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 640                              "413 Request Entity Too Large",
 641                              "SDMF is limited to one segment, and 10001 > 10000",
 642                              self.PUT,
 643                              self.public_url + "/foo/new.txt?mutable=true",
 644                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 645         return d
 646
 647     def test_PUT_NEWFILEURL_replace(self):
 648         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 649         # TODO: we lose the response code, so we can't check this
 650         #self.failUnlessEqual(responsecode, 200)
 651         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 652         d.addCallback(lambda res:
 653                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 654                                                       self.NEWFILE_CONTENTS))
 655         return d
 656
 657     def test_PUT_NEWFILEURL_bad_t(self):
 658         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 659                              "PUT to a file: bad t=bogus",
 660                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 661                              "contents")
 662         return d
 663
 664     def test_PUT_NEWFILEURL_no_replace(self):
 665         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 666                      self.NEWFILE_CONTENTS)
 667         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 668                   "409 Conflict",
 669                   "There was already a child by that name, and you asked me "
 670                   "to not replace it")
 671         return d
 672
 673     def test_PUT_NEWFILEURL_mkdirs(self):
 674         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 675         fn = self._foo_node
 676         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 677         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 678         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 679         d.addCallback(lambda res:
 680                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 681                                                       self.NEWFILE_CONTENTS))
 682         return d
 683
 684     def test_PUT_NEWFILEURL_blocked(self):
 685         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 686                      self.NEWFILE_CONTENTS)
 687         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 688                   "409 Conflict",
 689                   "Unable to create directory 'blockingfile': a file was in the way")
 690         return d
 691
 692     def test_DELETE_FILEURL(self):
 693         d = self.DELETE(self.public_url + "/foo/bar.txt")
 694         d.addCallback(lambda res:
 695                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 696         return d
 697
 698     def test_DELETE_FILEURL_missing(self):
 699         d = self.DELETE(self.public_url + "/foo/missing")
 700         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 701         return d
 702
 703     def test_DELETE_FILEURL_missing2(self):
 704         d = self.DELETE(self.public_url + "/missing/missing")
 705         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 706         return d
 707
 708     def test_GET_FILEURL_json(self):
 709         # twisted.web.http.parse_qs ignores any query args without an '=', so
 710         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 711         # instead. This may make it tricky to emulate the S3 interface
 712         # completely.
 713         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 714         d.addCallback(self.failUnlessIsBarJSON)
 715         return d
 716
 717     def test_GET_FILEURL_json_missing(self):
 718         d = self.GET(self.public_url + "/foo/missing?json")
 719         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 720         return d
 721
 722     def test_GET_FILEURL_uri(self):
 723         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 724         def _check(res):
 725             self.failUnlessEqual(res, self._bar_txt_uri)
 726         d.addCallback(_check)
 727         d.addCallback(lambda res:
 728                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 729         def _check2(res):
 730             # for now, for files, uris and readonly-uris are the same
 731             self.failUnlessEqual(res, self._bar_txt_uri)
 732         d.addCallback(_check2)
 733         return d
 734
 735     def test_GET_FILEURL_badtype(self):
 736         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 737                                   "bad t=bogus",
 738                                   self.GET,
 739                                   self.public_url + "/foo/bar.txt?t=bogus")
 740         return d
 741
 742     def test_GET_FILEURL_uri_missing(self):
 743         d = self.GET(self.public_url + "/foo/missing?t=uri")
 744         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 745         return d
 746
 747     def test_GET_DIRURL(self):
 748         # the addSlash means we get a redirect here
 749         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 750         ROOT = "../../.."
 751         d = self.GET(self.public_url + "/foo", followRedirect=True)
 752         def _check(res):
 753             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 754                             in res, res)
 755             # the FILE reference points to a URI, but it should end in bar.txt
 756             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 757                        (ROOT, urllib.quote(self._bar_txt_uri)))
 758             get_bar = "".join([r'<td>',
 759                                r'<a href="%s">bar.txt</a>' % bar_url,
 760                                r'</td>',
 761                                r'\s+<td>FILE</td>',
 762                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 763                                ])
 764             self.failUnless(re.search(get_bar, res), res)
 765             for line in res.split("\n"):
 766                 # find the line that contains the delete button for bar.txt
 767                 if ("form action" in line and
 768                     'value="delete"' in line and
 769                     'value="bar.txt"' in line):
 770                     # the form target should use a relative URL
 771                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 772                     self.failUnless(('action="%s"' % foo_url) in line, line)
 773                     # and the when_done= should too
 774                     #done_url = urllib.quote(???)
 775                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 776                     #                in line, line)
 777                     break
 778             else:
 779                 self.fail("unable to find delete-bar.txt line", res)
 780
 781             # the DIR reference just points to a URI
 782             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 783             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 784                        + r'\s+<td>DIR</td>')
 785             self.failUnless(re.search(get_sub, res), res)
 786         d.addCallback(_check)
 787
 788         # look at a directory which is readonly
 789         d.addCallback(lambda res:
 790                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 791         def _check2(res):
 792             self.failUnless("(readonly)" in res, res)
 793             self.failIf("Upload a file" in res, res)
 794         d.addCallback(_check2)
 795
 796         # and at a directory that contains a readonly directory
 797         d.addCallback(lambda res:
 798                       self.GET(self.public_url, followRedirect=True))
 799         def _check3(res):
 800             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 801                                       '</td>\s+<td>DIR-RO</td>', res))
 802         d.addCallback(_check3)
 803
 804         return d
 805
 806     def test_GET_DIRURL_badtype(self):
 807         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 808                                   400, "Bad Request",
 809                                   "bad t=bogus",
 810                                   self.GET,
 811                                   self.public_url + "/foo?t=bogus")
 812         return d
 813
 814     def test_GET_DIRURL_json(self):
 815         d = self.GET(self.public_url + "/foo?t=json")
 816         d.addCallback(self.failUnlessIsFooJSON)
 817         return d
 818
 819     def test_GET_DIRURL_manifest(self):
 820         d = self.GET(self.public_url + "/foo?t=manifest", followRedirect=True)
 821         def _got(manifest):
 822             self.failUnless("Refresh Capabilities" in manifest)
 823         d.addCallback(_got)
 824         return d
 825
 826     def test_GET_DIRURL_deepsize(self):
 827         d = self.GET(self.public_url + "/foo?t=deep-size", followRedirect=True)
 828         def _got(manifest):
 829             self.failUnless(re.search(r'^\d+$', manifest), manifest)
 830             self.failUnlessEqual("57", manifest)
 831         d.addCallback(_got)
 832         return d
 833
 834     def test_GET_DIRURL_deepstats(self):
 835         d = self.GET(self.public_url + "/foo?t=deep-stats", followRedirect=True)
 836         def _got(stats_json):
 837             stats = simplejson.loads(stats_json)
 838             expected = {"count-immutable-files": 3,
 839                         "count-mutable-files": 0,
 840                         "count-literal-files": 0,
 841                         "count-files": 3,
 842                         "count-directories": 3,
 843                         "size-immutable-files": 57,
 844                         "size-literal-files": 0,
 845                         #"size-directories": 1912, # varies
 846                         #"largest-directory": 1590,
 847                         "largest-directory-children": 5,
 848                         "largest-immutable-file": 19,
 849                         }
 850             for k,v in expected.iteritems():
 851                 self.failUnlessEqual(stats[k], v,
 852                                      "stats[%s] was %s, not %s" %
 853                                      (k, stats[k], v))
 854             self.failUnlessEqual(stats["size-files-histogram"],
 855                                  [ [11, 31, 3] ])
 856         d.addCallback(_got)
 857         return d
 858
 859     def test_GET_DIRURL_uri(self):
 860         d = self.GET(self.public_url + "/foo?t=uri")
 861         def _check(res):
 862             self.failUnlessEqual(res, self._foo_uri)
 863         d.addCallback(_check)
 864         return d
 865
 866     def test_GET_DIRURL_readonly_uri(self):
 867         d = self.GET(self.public_url + "/foo?t=readonly-uri")
 868         def _check(res):
 869             self.failUnlessEqual(res, self._foo_readonly_uri)
 870         d.addCallback(_check)
 871         return d
 872
 873     def test_PUT_NEWDIRURL(self):
 874         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
 875         d.addCallback(lambda res:
 876                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
 877         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
 878         d.addCallback(self.failUnlessNodeKeysAre, [])
 879         return d
 880
 881     def test_PUT_NEWDIRURL_exists(self):
 882         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
 883         d.addCallback(lambda res:
 884                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 885         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 886         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 887         return d
 888
 889     def test_PUT_NEWDIRURL_blocked(self):
 890         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
 891                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
 892                              self.PUT,
 893                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
 894         d.addCallback(lambda res:
 895                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
 896         d.addCallback(lambda res: self._foo_node.get(u"sub"))
 897         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
 898         return d
 899
 900     def test_PUT_NEWDIRURL_mkdir_p(self):
 901         d = defer.succeed(None)
 902         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
 903         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
 904         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
 905         def mkdir_p(mkpnode):
 906             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
 907             d = self.POST(url)
 908             def made_subsub(ssuri):
 909                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
 910                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
 911                 d = self.POST(url)
 912                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
 913                 return d
 914             d.addCallback(made_subsub)
 915             return d
 916         d.addCallback(mkdir_p)
 917         return d
 918
 919     def test_PUT_NEWDIRURL_mkdirs(self):
 920         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
 921         d.addCallback(lambda res:
 922                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
 923         d.addCallback(lambda res:
 924                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
 925         d.addCallback(lambda res:
 926                       self._foo_node.get_child_at_path(u"subdir/newdir"))
 927         d.addCallback(self.failUnlessNodeKeysAre, [])
 928         return d
 929
 930     def test_DELETE_DIRURL(self):
 931         d = self.DELETE(self.public_url + "/foo")
 932         d.addCallback(lambda res:
 933                       self.failIfNodeHasChild(self.public_root, u"foo"))
 934         return d
 935
 936     def test_DELETE_DIRURL_missing(self):
 937         d = self.DELETE(self.public_url + "/foo/missing")
 938         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
 939         d.addCallback(lambda res:
 940                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
 941         return d
 942
 943     def test_DELETE_DIRURL_missing2(self):
 944         d = self.DELETE(self.public_url + "/missing")
 945         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
 946         return d
 947
 948     def dump_root(self):
 949         print "NODEWALK"
 950         w = webish.DirnodeWalkerMixin()
 951         def visitor(childpath, childnode, metadata):
 952             print childpath
 953         d = w.walk(self.public_root, visitor)
 954         return d
 955
 956     def failUnlessNodeKeysAre(self, node, expected_keys):
 957         for k in expected_keys:
 958             assert isinstance(k, unicode)
 959         d = node.list()
 960         def _check(children):
 961             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
 962         d.addCallback(_check)
 963         return d
 964     def failUnlessNodeHasChild(self, node, name):
 965         assert isinstance(name, unicode)
 966         d = node.list()
 967         def _check(children):
 968             self.failUnless(name in children)
 969         d.addCallback(_check)
 970         return d
 971     def failIfNodeHasChild(self, node, name):
 972         assert isinstance(name, unicode)
 973         d = node.list()
 974         def _check(children):
 975             self.failIf(name in children)
 976         d.addCallback(_check)
 977         return d
 978
 979     def failUnlessChildContentsAre(self, node, name, expected_contents):
 980         assert isinstance(name, unicode)
 981         d = node.get_child_at_path(name)
 982         d.addCallback(lambda node: node.download_to_data())
 983         def _check(contents):
 984             self.failUnlessEqual(contents, expected_contents)
 985         d.addCallback(_check)
 986         return d
 987
 988     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
 989         assert isinstance(name, unicode)
 990         d = node.get_child_at_path(name)
 991         d.addCallback(lambda node: node.download_best_version())
 992         def _check(contents):
 993             self.failUnlessEqual(contents, expected_contents)
 994         d.addCallback(_check)
 995         return d
 996
 997     def failUnlessChildURIIs(self, node, name, expected_uri):
 998         assert isinstance(name, unicode)
 999         d = node.get_child_at_path(name)
1000         def _check(child):
1001             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1002         d.addCallback(_check)
1003         return d
1004
1005     def failUnlessURIMatchesChild(self, got_uri, node, name):
1006         assert isinstance(name, unicode)
1007         d = node.get_child_at_path(name)
1008         def _check(child):
1009             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1010         d.addCallback(_check)
1011         return d
1012
1013     def failUnlessCHKURIHasContents(self, got_uri, contents):
1014         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1015
1016     def test_POST_upload(self):
1017         d = self.POST(self.public_url + "/foo", t="upload",
1018                       file=("new.txt", self.NEWFILE_CONTENTS))
1019         fn = self._foo_node
1020         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1021         d.addCallback(lambda res:
1022                       self.failUnlessChildContentsAre(fn, u"new.txt",
1023                                                       self.NEWFILE_CONTENTS))
1024         return d
1025
1026     def test_POST_upload_unicode(self):
1027         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1028         d = self.POST(self.public_url + "/foo", t="upload",
1029                       file=(filename, self.NEWFILE_CONTENTS))
1030         fn = self._foo_node
1031         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1032         d.addCallback(lambda res:
1033                       self.failUnlessChildContentsAre(fn, filename,
1034                                                       self.NEWFILE_CONTENTS))
1035         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1036         d.addCallback(lambda res: self.GET(target_url))
1037         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1038                                                             self.NEWFILE_CONTENTS,
1039                                                             contents))
1040         return d
1041
1042     def test_POST_upload_unicode_named(self):
1043         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1044         d = self.POST(self.public_url + "/foo", t="upload",
1045                       name=filename,
1046                       file=("overridden", self.NEWFILE_CONTENTS))
1047         fn = self._foo_node
1048         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1049         d.addCallback(lambda res:
1050                       self.failUnlessChildContentsAre(fn, filename,
1051                                                       self.NEWFILE_CONTENTS))
1052         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1053         d.addCallback(lambda res: self.GET(target_url))
1054         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1055                                                             self.NEWFILE_CONTENTS,
1056                                                             contents))
1057         return d
1058
1059     def test_POST_upload_no_link(self):
1060         d = self.POST("/uri", t="upload",
1061                       file=("new.txt", self.NEWFILE_CONTENTS))
1062         def _check_upload_results(page):
1063             # this should be a page which describes the results of the upload
1064             # that just finished.
1065             self.failUnless("Upload Results:" in page)
1066             self.failUnless("URI:" in page)
1067             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1068             mo = uri_re.search(page)
1069             self.failUnless(mo, page)
1070             new_uri = mo.group(1)
1071             return new_uri
1072         d.addCallback(_check_upload_results)
1073         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1074         return d
1075
1076     def test_POST_upload_no_link_whendone(self):
1077         d = self.POST("/uri", t="upload", when_done="/",
1078                       file=("new.txt", self.NEWFILE_CONTENTS))
1079         d.addBoth(self.shouldRedirect, "/")
1080         return d
1081
1082     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1083         d = defer.maybeDeferred(callable, *args, **kwargs)
1084         def done(res):
1085             if isinstance(res, failure.Failure):
1086                 res.trap(error.PageRedirect)
1087                 statuscode = res.value.status
1088                 target = res.value.location
1089                 return checker(statuscode, target)
1090             self.fail("%s: callable was supposed to redirect, not return '%s'"
1091                       % (which, res))
1092         d.addBoth(done)
1093         return d
1094
1095     def test_POST_upload_no_link_whendone_results(self):
1096         def check(statuscode, target):
1097             self.failUnlessEqual(statuscode, str(http.FOUND))
1098             self.failUnless(target.startswith(self.webish_url), target)
1099             return client.getPage(target, method="GET")
1100         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1101                                  check,
1102                                  self.POST, "/uri", t="upload",
1103                                  when_done="/uri/%(uri)s",
1104                                  file=("new.txt", self.NEWFILE_CONTENTS))
1105         d.addCallback(lambda res:
1106                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1107         return d
1108
1109     def test_POST_upload_no_link_mutable(self):
1110         d = self.POST("/uri", t="upload", mutable="true",
1111                       file=("new.txt", self.NEWFILE_CONTENTS))
1112         def _check(new_uri):
1113             new_uri = new_uri.strip()
1114             self.new_uri = new_uri
1115             u = IURI(new_uri)
1116             self.failUnless(IMutableFileURI.providedBy(u))
1117             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1118             n = self.s.create_node_from_uri(new_uri)
1119             return n.download_best_version()
1120         d.addCallback(_check)
1121         def _check2(data):
1122             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1123             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1124         d.addCallback(_check2)
1125         def _check3(data):
1126             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1127             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1128         d.addCallback(_check3)
1129         def _check4(data):
1130             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1131         d.addCallback(_check4)
1132         return d
1133
1134     def test_POST_upload_no_link_mutable_toobig(self):
1135         d = self.shouldFail2(error.Error,
1136                              "test_POST_upload_no_link_mutable_toobig",
1137                              "413 Request Entity Too Large",
1138                              "SDMF is limited to one segment, and 10001 > 10000",
1139                              self.POST,
1140                              "/uri", t="upload", mutable="true",
1141                              file=("new.txt",
1142                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1143         return d
1144
1145     def test_POST_upload_mutable(self):
1146         # this creates a mutable file
1147         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1148                       file=("new.txt", self.NEWFILE_CONTENTS))
1149         fn = self._foo_node
1150         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1151         d.addCallback(lambda res:
1152                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1153                                                              self.NEWFILE_CONTENTS))
1154         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1155         def _got(newnode):
1156             self.failUnless(IMutableFileNode.providedBy(newnode))
1157             self.failUnless(newnode.is_mutable())
1158             self.failIf(newnode.is_readonly())
1159             self._mutable_node = newnode
1160             self._mutable_uri = newnode.get_uri()
1161         d.addCallback(_got)
1162
1163         # now upload it again and make sure that the URI doesn't change
1164         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1165         d.addCallback(lambda res:
1166                       self.POST(self.public_url + "/foo", t="upload",
1167                                 mutable="true",
1168                                 file=("new.txt", NEWER_CONTENTS)))
1169         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1170         d.addCallback(lambda res:
1171                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1172                                                              NEWER_CONTENTS))
1173         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1174         def _got2(newnode):
1175             self.failUnless(IMutableFileNode.providedBy(newnode))
1176             self.failUnless(newnode.is_mutable())
1177             self.failIf(newnode.is_readonly())
1178             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1179         d.addCallback(_got2)
1180
1181         # upload a second time, using PUT instead of POST
1182         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1183         d.addCallback(lambda res:
1184                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1185         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1186         d.addCallback(lambda res:
1187                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1188                                                              NEW2_CONTENTS))
1189
1190         # finally list the directory, since mutable files are displayed
1191         # differently
1192
1193         d.addCallback(lambda res:
1194                       self.GET(self.public_url + "/foo/",
1195                                followRedirect=True))
1196         def _check_page(res):
1197             # TODO: assert more about the contents
1198             self.failUnless("Overwrite" in res)
1199             self.failUnless("Choose new file:" in res)
1200             return res
1201         d.addCallback(_check_page)
1202
1203         # test that clicking on the "overwrite" button works
1204         EVEN_NEWER_CONTENTS = NEWER_CONTENTS + "even newer\n"
1205         def _parse_overwrite_form_and_submit(res):
1206
1207             OVERWRITE_FORM_RE=re.compile('<form action="([^"]*)" method="post" .*<input type="hidden" name="t" value="upload" /><input type="hidden" name="when_done" value="([^"]*)" />', re.I)
1208             mo = OVERWRITE_FORM_RE.search(res)
1209             self.failUnless(mo, "overwrite form not found in '" + res +
1210                             "', in which the overwrite form was not found")
1211             formaction=mo.group(1)
1212             formwhendone=mo.group(2)
1213
1214             fileurl = "../../../uri/" + urllib.quote(self._mutable_uri)
1215             self.failUnlessEqual(formaction, fileurl)
1216             # to POST, we need to absoluteify the URL
1217             new_formaction = "/uri/%s" % urllib.quote(self._mutable_uri)
1218             self.failUnlessEqual(formwhendone,
1219                                  "../uri/%s/" % urllib.quote(self._foo_uri))
1220             return self.POST(new_formaction,
1221                              t="upload",
1222                              file=("new.txt", EVEN_NEWER_CONTENTS),
1223                              when_done=formwhendone,
1224                              followRedirect=False)
1225         d.addCallback(_parse_overwrite_form_and_submit)
1226         # This will redirect us to ../uri/$FOOURI, rather than
1227         # ../uri/$PARENT/foo, but apparently twisted.web.client absolutifies
1228         # the redirect for us, and remember that shouldRedirect prepends
1229         # self.webish_url for us.
1230         d.addBoth(self.shouldRedirect,
1231                   "/uri/%s/" % urllib.quote(self._foo_uri),
1232                   which="test_POST_upload_mutable.overwrite")
1233         d.addCallback(lambda res:
1234                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1235                                                              EVEN_NEWER_CONTENTS))
1236         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1237         def _got3(newnode):
1238             self.failUnless(IMutableFileNode.providedBy(newnode))
1239             self.failUnless(newnode.is_mutable())
1240             self.failIf(newnode.is_readonly())
1241             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1242         d.addCallback(_got3)
1243
1244         # look at the JSON form of the enclosing directory
1245         d.addCallback(lambda res:
1246                       self.GET(self.public_url + "/foo/?t=json",
1247                                followRedirect=True))
1248         def _check_page_json(res):
1249             parsed = simplejson.loads(res)
1250             self.failUnlessEqual(parsed[0], "dirnode")
1251             children = parsed[1]["children"]
1252             self.failUnless("new.txt" in children)
1253             new_json = children["new.txt"]
1254             self.failUnlessEqual(new_json[0], "filenode")
1255             self.failUnless(new_json[1]["mutable"])
1256             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1257             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1258             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1259         d.addCallback(_check_page_json)
1260
1261         # and the JSON form of the file
1262         d.addCallback(lambda res:
1263                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1264         def _check_file_json(res):
1265             parsed = simplejson.loads(res)
1266             self.failUnlessEqual(parsed[0], "filenode")
1267             self.failUnless(parsed[1]["mutable"])
1268             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1269             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1270             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1271         d.addCallback(_check_file_json)
1272
1273         # and look at t=uri and t=readonly-uri
1274         d.addCallback(lambda res:
1275                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1276         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1277         d.addCallback(lambda res:
1278                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1279         def _check_ro_uri(res):
1280             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1281             self.failUnlessEqual(res, ro_uri)
1282         d.addCallback(_check_ro_uri)
1283
1284         # make sure we can get to it from /uri/URI
1285         d.addCallback(lambda res:
1286                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1287         d.addCallback(lambda res:
1288                       self.failUnlessEqual(res, EVEN_NEWER_CONTENTS))
1289
1290         # and that HEAD computes the size correctly
1291         d.addCallback(lambda res:
1292                       self.HEAD(self.public_url + "/foo/new.txt"))
1293         def _got_headers(headers):
1294             self.failUnlessEqual(headers["content-length"][0],
1295                                  str(len(EVEN_NEWER_CONTENTS)))
1296             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1297         d.addCallback(_got_headers)
1298
1299         # make sure that size errors are displayed correctly for overwrite
1300         d.addCallback(lambda res:
1301                       self.shouldFail2(error.Error,
1302                                        "test_POST_upload_mutable-toobig",
1303                                        "413 Request Entity Too Large",
1304                                        "SDMF is limited to one segment, and 10001 > 10000",
1305                                        self.POST,
1306                                        self.public_url + "/foo", t="upload",
1307                                        mutable="true",
1308                                        file=("new.txt",
1309                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1310                                        ))
1311
1312         d.addErrback(self.dump_error)
1313         return d
1314
1315     def test_POST_upload_mutable_toobig(self):
1316         d = self.shouldFail2(error.Error,
1317                              "test_POST_upload_no_link_mutable_toobig",
1318                              "413 Request Entity Too Large",
1319                              "SDMF is limited to one segment, and 10001 > 10000",
1320                              self.POST,
1321                              self.public_url + "/foo",
1322                              t="upload", mutable="true",
1323                              file=("new.txt",
1324                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1325         return d
1326
1327     def dump_error(self, f):
1328         # if the web server returns an error code (like 400 Bad Request),
1329         # web.client.getPage puts the HTTP response body into the .response
1330         # attribute of the exception object that it gives back. It does not
1331         # appear in the Failure's repr(), so the ERROR that trial displays
1332         # will be rather terse and unhelpful. addErrback this method to the
1333         # end of your chain to get more information out of these errors.
1334         if f.check(error.Error):
1335             print "web.error.Error:"
1336             print f
1337             print f.value.response
1338         return f
1339
1340     def test_POST_upload_replace(self):
1341         d = self.POST(self.public_url + "/foo", t="upload",
1342                       file=("bar.txt", self.NEWFILE_CONTENTS))
1343         fn = self._foo_node
1344         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1345         d.addCallback(lambda res:
1346                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1347                                                       self.NEWFILE_CONTENTS))
1348         return d
1349
1350     def test_POST_upload_no_replace_ok(self):
1351         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1352                       file=("new.txt", self.NEWFILE_CONTENTS))
1353         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1354         d.addCallback(lambda res: self.failUnlessEqual(res,
1355                                                        self.NEWFILE_CONTENTS))
1356         return d
1357
1358     def test_POST_upload_no_replace_queryarg(self):
1359         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1360                       file=("bar.txt", self.NEWFILE_CONTENTS))
1361         d.addBoth(self.shouldFail, error.Error,
1362                   "POST_upload_no_replace_queryarg",
1363                   "409 Conflict",
1364                   "There was already a child by that name, and you asked me "
1365                   "to not replace it")
1366         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1367         d.addCallback(self.failUnlessIsBarDotTxt)
1368         return d
1369
1370     def test_POST_upload_no_replace_field(self):
1371         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1372                       file=("bar.txt", self.NEWFILE_CONTENTS))
1373         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1374                   "409 Conflict",
1375                   "There was already a child by that name, and you asked me "
1376                   "to not replace it")
1377         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1378         d.addCallback(self.failUnlessIsBarDotTxt)
1379         return d
1380
1381     def test_POST_upload_whendone(self):
1382         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1383                       file=("new.txt", self.NEWFILE_CONTENTS))
1384         d.addBoth(self.shouldRedirect, "/THERE")
1385         fn = self._foo_node
1386         d.addCallback(lambda res:
1387                       self.failUnlessChildContentsAre(fn, u"new.txt",
1388                                                       self.NEWFILE_CONTENTS))
1389         return d
1390
1391     def test_POST_upload_named(self):
1392         fn = self._foo_node
1393         d = self.POST(self.public_url + "/foo", t="upload",
1394                       name="new.txt", file=self.NEWFILE_CONTENTS)
1395         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1396         d.addCallback(lambda res:
1397                       self.failUnlessChildContentsAre(fn, u"new.txt",
1398                                                       self.NEWFILE_CONTENTS))
1399         return d
1400
1401     def test_POST_upload_named_badfilename(self):
1402         d = self.POST(self.public_url + "/foo", t="upload",
1403                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1404         d.addBoth(self.shouldFail, error.Error,
1405                   "test_POST_upload_named_badfilename",
1406                   "400 Bad Request",
1407                   "name= may not contain a slash",
1408                   )
1409         # make sure that nothing was added
1410         d.addCallback(lambda res:
1411                       self.failUnlessNodeKeysAre(self._foo_node,
1412                                                  [u"bar.txt", u"blockingfile",
1413                                                   u"empty", u"n\u00fc.txt",
1414                                                   u"sub"]))
1415         return d
1416
1417     def test_POST_FILEURL_check(self):
1418         bar_url = self.public_url + "/foo/bar.txt"
1419         d = self.POST(bar_url, t="check")
1420         def _check(res):
1421             self.failUnless("Healthy!" in res)
1422         d.addCallback(_check)
1423         redir_url = "http://allmydata.org/TARGET"
1424         def _check2(statuscode, target):
1425             self.failUnlessEqual(statuscode, str(http.FOUND))
1426             self.failUnlessEqual(target, redir_url)
1427         d.addCallback(lambda res:
1428                       self.shouldRedirect2("test_POST_FILEURL_check",
1429                                            _check2,
1430                                            self.POST, bar_url,
1431                                            t="check",
1432                                            when_done=redir_url))
1433         d.addCallback(lambda res:
1434                       self.POST(bar_url, t="check", return_to=redir_url))
1435         def _check3(res):
1436             self.failUnless("Healthy!" in res)
1437             self.failUnless("Return to parent directory" in res)
1438             self.failUnless(redir_url in res)
1439         d.addCallback(_check3)
1440         return d
1441
1442     def test_POST_DIRURL_check(self):
1443         foo_url = self.public_url + "/foo/"
1444         d = self.POST(foo_url, t="check")
1445         def _check(res):
1446             self.failUnless("Healthy!" in res)
1447         d.addCallback(_check)
1448         redir_url = "http://allmydata.org/TARGET"
1449         def _check2(statuscode, target):
1450             self.failUnlessEqual(statuscode, str(http.FOUND))
1451             self.failUnlessEqual(target, redir_url)
1452         d.addCallback(lambda res:
1453                       self.shouldRedirect2("test_POST_DIRURL_check",
1454                                            _check2,
1455                                            self.POST, foo_url,
1456                                            t="check",
1457                                            when_done=redir_url))
1458         d.addCallback(lambda res:
1459                       self.POST(foo_url, t="check", return_to=redir_url))
1460         def _check3(res):
1461             self.failUnless("Healthy!" in res)
1462             self.failUnless("Return to parent directory" in res)
1463             self.failUnless(redir_url in res)
1464         d.addCallback(_check3)
1465         return d
1466
1467     def test_POST_DIRURL_deepcheck(self):
1468         d = self.POST(self.public_url, t="deep-check")
1469         def _check(res):
1470             self.failUnless("Objects Checked: <span>8</span>" in res)
1471             self.failUnless("Objects Healthy: <span>8</span>" in res)
1472             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1473             self.failUnless("Repairs Successful: <span>0</span>" in res)
1474         d.addCallback(_check)
1475         redir_url = "http://allmydata.org/TARGET"
1476         def _check2(statuscode, target):
1477             self.failUnlessEqual(statuscode, str(http.FOUND))
1478             self.failUnlessEqual(target, redir_url)
1479         d.addCallback(lambda res:
1480                       self.shouldRedirect2("test_POST_DIRURL_check",
1481                                            _check2,
1482                                            self.POST, self.public_url,
1483                                            t="deep-check",
1484                                            when_done=redir_url))
1485         d.addCallback(lambda res:
1486                       self.POST(self.public_url, t="deep-check",
1487                                 return_to=redir_url))
1488         def _check3(res):
1489             self.failUnless("Return to parent directory" in res)
1490             self.failUnless(redir_url in res)
1491         d.addCallback(_check3)
1492         return d
1493
1494     def test_POST_FILEURL_bad_t(self):
1495         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1496                              "POST to file: bad t=bogus",
1497                              self.POST, self.public_url + "/foo/bar.txt",
1498                              t="bogus")
1499         return d
1500
1501     def test_POST_mkdir(self): # return value?
1502         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1503         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1504         d.addCallback(self.failUnlessNodeKeysAre, [])
1505         return d
1506
1507     def test_POST_mkdir_2(self):
1508         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1509         d.addCallback(lambda res:
1510                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1511         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1512         d.addCallback(self.failUnlessNodeKeysAre, [])
1513         return d
1514
1515     def test_POST_mkdirs_2(self):
1516         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1517         d.addCallback(lambda res:
1518                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1519         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1520         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1521         d.addCallback(self.failUnlessNodeKeysAre, [])
1522         return d
1523
1524     def test_POST_mkdir_no_parentdir_noredirect(self):
1525         d = self.POST("/uri?t=mkdir")
1526         def _after_mkdir(res):
1527             uri.NewDirectoryURI.init_from_string(res)
1528         d.addCallback(_after_mkdir)
1529         return d
1530
1531     def test_POST_mkdir_no_parentdir_redirect(self):
1532         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1533         d.addBoth(self.shouldRedirect, None, statuscode='303')
1534         def _check_target(target):
1535             target = urllib.unquote(target)
1536             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1537         d.addCallback(_check_target)
1538         return d
1539
1540     def test_POST_noparent_bad(self):
1541         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1542                                   "/uri accepts only PUT, PUT?t=mkdir, "
1543                                   "POST?t=upload, and POST?t=mkdir",
1544                                   self.POST, "/uri?t=bogus")
1545         return d
1546
1547     def test_welcome_page_mkdir_button(self):
1548         # Fetch the welcome page.
1549         d = self.GET("/")
1550         def _after_get_welcome_page(res):
1551             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1552             mo = MKDIR_BUTTON_RE.search(res)
1553             formaction = mo.group(1)
1554             formt = mo.group(2)
1555             formaname = mo.group(3)
1556             formavalue = mo.group(4)
1557             return (formaction, formt, formaname, formavalue)
1558         d.addCallback(_after_get_welcome_page)
1559         def _after_parse_form(res):
1560             (formaction, formt, formaname, formavalue) = res
1561             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1562         d.addCallback(_after_parse_form)
1563         d.addBoth(self.shouldRedirect, None, statuscode='303')
1564         return d
1565
1566     def test_POST_mkdir_replace(self): # return value?
1567         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1568         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1569         d.addCallback(self.failUnlessNodeKeysAre, [])
1570         return d
1571
1572     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1573         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1574         d.addBoth(self.shouldFail, error.Error,
1575                   "POST_mkdir_no_replace_queryarg",
1576                   "409 Conflict",
1577                   "There was already a child by that name, and you asked me "
1578                   "to not replace it")
1579         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1580         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1581         return d
1582
1583     def test_POST_mkdir_no_replace_field(self): # return value?
1584         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1585                       replace="false")
1586         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1587                   "409 Conflict",
1588                   "There was already a child by that name, and you asked me "
1589                   "to not replace it")
1590         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1591         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1592         return d
1593
1594     def test_POST_mkdir_whendone_field(self):
1595         d = self.POST(self.public_url + "/foo",
1596                       t="mkdir", name="newdir", when_done="/THERE")
1597         d.addBoth(self.shouldRedirect, "/THERE")
1598         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1599         d.addCallback(self.failUnlessNodeKeysAre, [])
1600         return d
1601
1602     def test_POST_mkdir_whendone_queryarg(self):
1603         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1604                       t="mkdir", name="newdir")
1605         d.addBoth(self.shouldRedirect, "/THERE")
1606         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1607         d.addCallback(self.failUnlessNodeKeysAre, [])
1608         return d
1609
1610     def test_POST_bad_t(self):
1611         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1612                              "POST to a directory with bad t=BOGUS",
1613                              self.POST, self.public_url + "/foo", t="BOGUS")
1614         return d
1615
1616     def test_POST_set_children(self):
1617         contents9, n9, newuri9 = self.makefile(9)
1618         contents10, n10, newuri10 = self.makefile(10)
1619         contents11, n11, newuri11 = self.makefile(11)
1620
1621         reqbody = """{
1622                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1623                                                 "size": 0,
1624                                                 "metadata": {
1625                                                   "ctime": 1002777696.7564139,
1626                                                   "mtime": 1002777696.7564139
1627                                                  }
1628                                                } ],
1629                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1630                                                 "size": 1,
1631                                                 "metadata": {
1632                                                   "ctime": 1002777696.7564139,
1633                                                   "mtime": 1002777696.7564139
1634                                                  }
1635                                                } ],
1636                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1637                                                 "size": 2,
1638                                                 "metadata": {
1639                                                   "ctime": 1002777696.7564139,
1640                                                   "mtime": 1002777696.7564139
1641                                                  }
1642                                                } ]
1643                     }""" % (newuri9, newuri10, newuri11)
1644
1645         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1646
1647         d = client.getPage(url, method="POST", postdata=reqbody)
1648         def _then(res):
1649             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1650             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1651             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1652
1653         d.addCallback(_then)
1654         d.addErrback(self.dump_error)
1655         return d
1656
1657     def test_POST_put_uri(self):
1658         contents, n, newuri = self.makefile(8)
1659         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1660         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1661         d.addCallback(lambda res:
1662                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1663                                                       contents))
1664         return d
1665
1666     def test_POST_put_uri_replace(self):
1667         contents, n, newuri = self.makefile(8)
1668         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1669         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1670         d.addCallback(lambda res:
1671                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1672                                                       contents))
1673         return d
1674
1675     def test_POST_put_uri_no_replace_queryarg(self):
1676         contents, n, newuri = self.makefile(8)
1677         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
1678                       name="bar.txt", uri=newuri)
1679         d.addBoth(self.shouldFail, error.Error,
1680                   "POST_put_uri_no_replace_queryarg",
1681                   "409 Conflict",
1682                   "There was already a child by that name, and you asked me "
1683                   "to not replace it")
1684         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1685         d.addCallback(self.failUnlessIsBarDotTxt)
1686         return d
1687
1688     def test_POST_put_uri_no_replace_field(self):
1689         contents, n, newuri = self.makefile(8)
1690         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
1691                       name="bar.txt", uri=newuri)
1692         d.addBoth(self.shouldFail, error.Error,
1693                   "POST_put_uri_no_replace_field",
1694                   "409 Conflict",
1695                   "There was already a child by that name, and you asked me "
1696                   "to not replace it")
1697         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1698         d.addCallback(self.failUnlessIsBarDotTxt)
1699         return d
1700
1701     def test_POST_delete(self):
1702         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
1703         d.addCallback(lambda res: self._foo_node.list())
1704         def _check(children):
1705             self.failIf(u"bar.txt" in children)
1706         d.addCallback(_check)
1707         return d
1708
1709     def test_POST_rename_file(self):
1710         d = self.POST(self.public_url + "/foo", t="rename",
1711                       from_name="bar.txt", to_name='wibble.txt')
1712         d.addCallback(lambda res:
1713                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1714         d.addCallback(lambda res:
1715                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
1716         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
1717         d.addCallback(self.failUnlessIsBarDotTxt)
1718         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
1719         d.addCallback(self.failUnlessIsBarJSON)
1720         return d
1721
1722     def test_POST_rename_file_replace(self):
1723         # rename a file and replace a directory with it
1724         d = self.POST(self.public_url + "/foo", t="rename",
1725                       from_name="bar.txt", to_name='empty')
1726         d.addCallback(lambda res:
1727                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1728         d.addCallback(lambda res:
1729                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
1730         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
1731         d.addCallback(self.failUnlessIsBarDotTxt)
1732         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1733         d.addCallback(self.failUnlessIsBarJSON)
1734         return d
1735
1736     def test_POST_rename_file_no_replace_queryarg(self):
1737         # rename a file and replace a directory with it
1738         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
1739                       from_name="bar.txt", to_name='empty')
1740         d.addBoth(self.shouldFail, error.Error,
1741                   "POST_rename_file_no_replace_queryarg",
1742                   "409 Conflict",
1743                   "There was already a child by that name, and you asked me "
1744                   "to not replace it")
1745         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1746         d.addCallback(self.failUnlessIsEmptyJSON)
1747         return d
1748
1749     def test_POST_rename_file_no_replace_field(self):
1750         # rename a file and replace a directory with it
1751         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
1752                       from_name="bar.txt", to_name='empty')
1753         d.addBoth(self.shouldFail, error.Error,
1754                   "POST_rename_file_no_replace_field",
1755                   "409 Conflict",
1756                   "There was already a child by that name, and you asked me "
1757                   "to not replace it")
1758         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
1759         d.addCallback(self.failUnlessIsEmptyJSON)
1760         return d
1761
1762     def failUnlessIsEmptyJSON(self, res):
1763         data = simplejson.loads(res)
1764         self.failUnlessEqual(data[0], "dirnode", data)
1765         self.failUnlessEqual(len(data[1]["children"]), 0)
1766
1767     def test_POST_rename_file_slash_fail(self):
1768         d = self.POST(self.public_url + "/foo", t="rename",
1769                       from_name="bar.txt", to_name='kirk/spock.txt')
1770         d.addBoth(self.shouldFail, error.Error,
1771                   "test_POST_rename_file_slash_fail",
1772                   "400 Bad Request",
1773                   "to_name= may not contain a slash",
1774                   )
1775         d.addCallback(lambda res:
1776                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1777         d.addCallback(lambda res: self.POST(self.public_url, t="rename",
1778                       from_name="foo/bar.txt", to_name='george.txt'))
1779         d.addBoth(self.shouldFail, error.Error,
1780                   "test_POST_rename_file_slash_fail",
1781                   "400 Bad Request",
1782                   "from_name= may not contain a slash",
1783                   )
1784         d.addCallback(lambda res:
1785                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1786         d.addCallback(lambda res:
1787                       self.failIfNodeHasChild(self.public_root, u"george.txt"))
1788         d.addCallback(lambda res:
1789                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
1790         d.addCallback(lambda res: self.GET(self.public_url + "/foo?t=json"))
1791         d.addCallback(self.failUnlessIsFooJSON)
1792         return d
1793
1794     def test_POST_rename_dir(self):
1795         d = self.POST(self.public_url, t="rename",
1796                       from_name="foo", to_name='plunk')
1797         d.addCallback(lambda res:
1798                       self.failIfNodeHasChild(self.public_root, u"foo"))
1799         d.addCallback(lambda res:
1800                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
1801         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
1802         d.addCallback(self.failUnlessIsFooJSON)
1803         return d
1804
1805     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
1806         """ If target is not None then the redirection has to go to target.  If
1807         statuscode is not None then the redirection has to be accomplished with
1808         that HTTP status code."""
1809         if not isinstance(res, failure.Failure):
1810             to_where = (target is None) and "somewhere" or ("to " + target)
1811             self.fail("%s: we were expecting to get redirected %s, not get an"
1812                       " actual page: %s" % (which, to_where, res))
1813         res.trap(error.PageRedirect)
1814         if statuscode is not None:
1815             self.failUnlessEqual(res.value.status, statuscode,
1816                                  "%s: not a redirect" % which)
1817         if target is not None:
1818             # the PageRedirect does not seem to capture the uri= query arg
1819             # properly, so we can't check for it.
1820             realtarget = self.webish_url + target
1821             self.failUnlessEqual(res.value.location, realtarget,
1822                                  "%s: wrong target" % which)
1823         return res.value.location
1824
1825     def test_GET_URI_form(self):
1826         base = "/uri?uri=%s" % self._bar_txt_uri
1827         # this is supposed to give us a redirect to /uri/$URI, plus arguments
1828         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1829         d = self.GET(base)
1830         d.addBoth(self.shouldRedirect, targetbase)
1831         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
1832         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
1833         d.addCallback(lambda res: self.GET(base+"&t=json"))
1834         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
1835         d.addCallback(self.log, "about to get file by uri")
1836         d.addCallback(lambda res: self.GET(base, followRedirect=True))
1837         d.addCallback(self.failUnlessIsBarDotTxt)
1838         d.addCallback(self.log, "got file by uri, about to get dir by uri")
1839         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
1840                                            followRedirect=True))
1841         d.addCallback(self.failUnlessIsFooJSON)
1842         d.addCallback(self.log, "got dir by uri")
1843
1844         return d
1845
1846     def test_GET_URI_form_bad(self):
1847         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
1848                              "400 Bad Request", "GET /uri requires uri=",
1849                              self.GET, "/uri")
1850         return d
1851
1852     def test_GET_rename_form(self):
1853         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
1854                      followRedirect=True)
1855         def _check(res):
1856             self.failUnless('name="when_done" value="."' in res, res)
1857             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
1858         d.addCallback(_check)
1859         return d
1860
1861     def log(self, res, msg):
1862         #print "MSG: %s  RES: %s" % (msg, res)
1863         log.msg(msg)
1864         return res
1865
1866     def test_GET_URI_URL(self):
1867         base = "/uri/%s" % self._bar_txt_uri
1868         d = self.GET(base)
1869         d.addCallback(self.failUnlessIsBarDotTxt)
1870         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
1871         d.addCallback(self.failUnlessIsBarDotTxt)
1872         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
1873         d.addCallback(self.failUnlessIsBarDotTxt)
1874         return d
1875
1876     def test_GET_URI_URL_dir(self):
1877         base = "/uri/%s?t=json" % self._foo_uri
1878         d = self.GET(base)
1879         d.addCallback(self.failUnlessIsFooJSON)
1880         return d
1881
1882     def test_GET_URI_URL_missing(self):
1883         base = "/uri/%s" % self._bad_file_uri
1884         d = self.GET(base)
1885         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
1886                   http.GONE, response_substring="NotEnoughSharesError")
1887         # TODO: how can we exercise both sides of WebDownloadTarget.fail
1888         # here? we must arrange for a download to fail after target.open()
1889         # has been called, and then inspect the response to see that it is
1890         # shorter than we expected.
1891         return d
1892
1893     def test_PUT_NEWFILEURL_uri(self):
1894         contents, n, new_uri = self.makefile(8)
1895         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
1896         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1897         d.addCallback(lambda res:
1898                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1899                                                       contents))
1900         return d
1901
1902     def test_PUT_NEWFILEURL_uri_replace(self):
1903         contents, n, new_uri = self.makefile(8)
1904         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
1905         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
1906         d.addCallback(lambda res:
1907                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1908                                                       contents))
1909         return d
1910
1911     def test_PUT_NEWFILEURL_uri_no_replace(self):
1912         contents, n, new_uri = self.makefile(8)
1913         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
1914         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
1915                   "409 Conflict",
1916                   "There was already a child by that name, and you asked me "
1917                   "to not replace it")
1918         return d
1919
1920     def test_PUT_NEWFILE_URI(self):
1921         file_contents = "New file contents here\n"
1922         d = self.PUT("/uri", file_contents)
1923         def _check(uri):
1924             self.failUnless(uri in FakeCHKFileNode.all_contents)
1925             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
1926                                  file_contents)
1927             return self.GET("/uri/%s" % uri)
1928         d.addCallback(_check)
1929         def _check2(res):
1930             self.failUnlessEqual(res, file_contents)
1931         d.addCallback(_check2)
1932         return d
1933
1934     def test_PUT_NEWFILE_URI_only_PUT(self):
1935         d = self.PUT("/uri?t=bogus", "")
1936         d.addBoth(self.shouldFail, error.Error,
1937                   "PUT_NEWFILE_URI_only_PUT",
1938                   "400 Bad Request",
1939                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
1940         return d
1941
1942     def test_PUT_NEWFILE_URI_mutable(self):
1943         file_contents = "New file contents here\n"
1944         d = self.PUT("/uri?mutable=true", file_contents)
1945         def _check_mutable(uri):
1946             uri = uri.strip()
1947             u = IURI(uri)
1948             self.failUnless(IMutableFileURI.providedBy(u))
1949             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1950             n = self.s.create_node_from_uri(uri)
1951             return n.download_best_version()
1952         d.addCallback(_check_mutable)
1953         def _check2_mutable(data):
1954             self.failUnlessEqual(data, file_contents)
1955         d.addCallback(_check2_mutable)
1956         return d
1957
1958         def _check(uri):
1959             self.failUnless(uri in FakeCHKFileNode.all_contents)
1960             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
1961                                  file_contents)
1962             return self.GET("/uri/%s" % uri)
1963         d.addCallback(_check)
1964         def _check2(res):
1965             self.failUnlessEqual(res, file_contents)
1966         d.addCallback(_check2)
1967         return d
1968
1969     def test_PUT_mkdir(self):
1970         d = self.PUT("/uri?t=mkdir", "")
1971         def _check(uri):
1972             n = self.s.create_node_from_uri(uri.strip())
1973             d2 = self.failUnlessNodeKeysAre(n, [])
1974             d2.addCallback(lambda res:
1975                            self.GET("/uri/%s?t=json" % uri))
1976             return d2
1977         d.addCallback(_check)
1978         d.addCallback(self.failUnlessIsEmptyJSON)
1979         return d
1980
1981     def test_POST_check(self):
1982         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
1983         def _done(res):
1984             # this returns a string form of the results, which are probably
1985             # None since we're using fake filenodes.
1986             # TODO: verify that the check actually happened, by changing
1987             # FakeCHKFileNode to count how many times .check() has been
1988             # called.
1989             pass
1990         d.addCallback(_done)
1991         return d
1992
1993     def test_bad_method(self):
1994         url = self.webish_url + self.public_url + "/foo/bar.txt"
1995         d = self.shouldHTTPError2("test_bad_method",
1996                                   501, "Not Implemented",
1997                                   "I don't know how to treat a BOGUS request.",
1998                                   client.getPage, url, method="BOGUS")
1999         return d
2000
2001     def test_short_url(self):
2002         url = self.webish_url + "/uri"
2003         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2004                                   "I don't know how to treat a DELETE request.",
2005                                   client.getPage, url, method="DELETE")
2006         return d
2007
2008 class Util(unittest.TestCase):
2009     def test_abbreviate_time(self):
2010         self.failUnlessEqual(common.abbreviate_time(None), "")
2011         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2012         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2013         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2014         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2015
2016     def test_abbreviate_rate(self):
2017         self.failUnlessEqual(common.abbreviate_rate(None), "")
2018         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2019         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2020         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2021
2022     def test_abbreviate_size(self):
2023         self.failUnlessEqual(common.abbreviate_size(None), "")
2024         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2025         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2026         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2027         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2028