src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from allmydata import interfaces, uri, webish
  10 from allmydata.storage.mutable import MutableShareFile
  11 from allmydata.storage.immutable import ShareFile
  12 from allmydata.immutable import upload, download
  13 from allmydata.web import status, common
  14 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  15 from allmydata.util import fileutil, base32
  16 from allmydata.util.assertutil import precondition
  17 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  18      FakeMutableFileNode, create_chk_filenode, WebErrorMixin
  19 from allmydata.interfaces import IURI, INewDirectoryURI, \
  20      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  21 from allmydata.mutable import servermap, publish, retrieve
  22 import common_util as testutil
  23 from allmydata.test.no_network import GridTestMixin
  24
  25 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  26 # create a webserver that works against them
  27
  28 class FakeIntroducerClient:
  29     def get_all_connectors(self):
  30         return {}
  31     def get_all_connections_for(self, service_name):
  32         return frozenset()
  33     def get_all_peerids(self):
  34         return frozenset()
  35
  36 class FakeClient(service.MultiService):
  37     nodeid = "fake_nodeid"
  38     nickname = "fake_nickname"
  39     basedir = "fake_basedir"
  40     def get_versions(self):
  41         return {'allmydata': "fake",
  42                 'foolscap': "fake",
  43                 'twisted': "fake",
  44                 'zfec': "fake",
  45                 }
  46     introducer_furl = "None"
  47     introducer_client = FakeIntroducerClient()
  48     _all_upload_status = [upload.UploadStatus()]
  49     _all_download_status = [download.DownloadStatus()]
  50     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  51     _all_publish_statuses = [publish.PublishStatus()]
  52     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  53     convergence = "some random string"
  54
  55     def connected_to_introducer(self):
  56         return False
  57
  58     def get_nickname_for_peerid(self, peerid):
  59         return u"John Doe"
  60
  61     def get_permuted_peers(self, service_name, key):
  62         return []
  63
  64     def create_node_from_uri(self, auri):
  65         precondition(isinstance(auri, str), auri)
  66         u = uri.from_string(auri)
  67         if (INewDirectoryURI.providedBy(u)
  68             or IReadonlyNewDirectoryURI.providedBy(u)):
  69             return FakeDirectoryNode(self).init_from_uri(u)
  70         if IFileURI.providedBy(u):
  71             return FakeCHKFileNode(u, self)
  72         assert IMutableFileURI.providedBy(u), u
  73         return FakeMutableFileNode(self).init_from_uri(u)
  74
  75     def create_empty_dirnode(self):
  76         n = FakeDirectoryNode(self)
  77         d = n.create()
  78         d.addCallback(lambda res: n)
  79         return d
  80
  81     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  82     def create_mutable_file(self, contents=""):
  83         n = FakeMutableFileNode(self)
  84         return n.create(contents)
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(self, data)
  92             results = upload.UploadResults()
  93             results.uri = n.get_uri()
  94             return results
  95         d.addCallback(_got_data)
  96         return d
  97
  98     def list_all_upload_statuses(self):
  99         return self._all_upload_status
 100     def list_all_download_statuses(self):
 101         return self._all_download_status
 102     def list_all_mapupdate_statuses(self):
 103         return self._all_mapupdate_statuses
 104     def list_all_publish_statuses(self):
 105         return self._all_publish_statuses
 106     def list_all_retrieve_statuses(self):
 107         return self._all_retrieve_statuses
 108     def list_all_helper_statuses(self):
 109         return []
 110
 111 class MyGetter(client.HTTPPageGetter):
 112     handleStatus_206 = lambda self: self.handleStatus_200()
 113
 114 class HTTPClientHEADFactory(client.HTTPClientFactory):
 115     protocol = MyGetter
 116
 117     def noPage(self, reason):
 118         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 119         # exception when the response to a HEAD request had no body (when in
 120         # fact they are defined to never have a body). This was fixed in
 121         # Twisted-8.0 . To work around this, we catch the
 122         # PartialDownloadError and make it disappear.
 123         if (reason.check(client.PartialDownloadError)
 124             and self.method.upper() == "HEAD"):
 125             self.page("")
 126             return
 127         return client.HTTPClientFactory.noPage(self, reason)
 128
 129 class HTTPClientGETFactory(client.HTTPClientFactory):
 130     protocol = MyGetter
 131
 132 class WebMixin(object):
 133     def setUp(self):
 134         self.s = FakeClient()
 135         self.s.startService()
 136         self.staticdir = self.mktemp()
 137         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 138         s.setServiceParent(self.s)
 139         self.webish_port = port = s.listener._port.getHost().port
 140         self.webish_url = "http://localhost:%d" % port
 141
 142         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 143         d = defer.DeferredList(l)
 144         def _then(res):
 145             self.public_root = res[0][1]
 146             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 147             self.public_url = "/uri/" + self.public_root.get_uri()
 148             self.private_root = res[1][1]
 149
 150             foo = res[2][1]
 151             self._foo_node = foo
 152             self._foo_uri = foo.get_uri()
 153             self._foo_readonly_uri = foo.get_readonly_uri()
 154             self._foo_verifycap = foo.get_verify_cap().to_string()
 155             # NOTE: we ignore the deferred on all set_uri() calls, because we
 156             # know the fake nodes do these synchronously
 157             self.public_root.set_uri(u"foo", foo.get_uri())
 158
 159             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 160             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 161             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 162
 163             foo.set_uri(u"empty", res[3][1].get_uri())
 164             sub_uri = res[4][1].get_uri()
 165             self._sub_uri = sub_uri
 166             foo.set_uri(u"sub", sub_uri)
 167             sub = self.s.create_node_from_uri(sub_uri)
 168
 169             _ign, n, blocking_uri = self.makefile(1)
 170             foo.set_uri(u"blockingfile", blocking_uri)
 171
 172             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 173             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 174             # still think of it as an umlaut
 175             foo.set_uri(unicode_filename, self._bar_txt_uri)
 176
 177             _ign, n, baz_file = self.makefile(2)
 178             self._baz_file_uri = baz_file
 179             sub.set_uri(u"baz.txt", baz_file)
 180
 181             _ign, n, self._bad_file_uri = self.makefile(3)
 182             # this uri should not be downloadable
 183             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 184
 185             rodir = res[5][1]
 186             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 187             rodir.set_uri(u"nor", baz_file)
 188
 189             # public/
 190             # public/foo/
 191             # public/foo/bar.txt
 192             # public/foo/blockingfile
 193             # public/foo/empty/
 194             # public/foo/sub/
 195             # public/foo/sub/baz.txt
 196             # public/reedownlee/
 197             # public/reedownlee/nor
 198             self.NEWFILE_CONTENTS = "newfile contents\n"
 199
 200             return foo.get_metadata_for(u"bar.txt")
 201         d.addCallback(_then)
 202         def _got_metadata(metadata):
 203             self._bar_txt_metadata = metadata
 204         d.addCallback(_got_metadata)
 205         return d
 206
 207     def makefile(self, number):
 208         contents = "contents of file %s\n" % number
 209         n = create_chk_filenode(self.s, contents)
 210         return contents, n, n.get_uri()
 211
 212     def tearDown(self):
 213         return self.s.stopService()
 214
 215     def failUnlessIsBarDotTxt(self, res):
 216         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 217
 218     def failUnlessIsBarJSON(self, res):
 219         data = simplejson.loads(res)
 220         self.failUnless(isinstance(data, list))
 221         self.failUnlessEqual(data[0], u"filenode")
 222         self.failUnless(isinstance(data[1], dict))
 223         self.failIf(data[1]["mutable"])
 224         self.failIf("rw_uri" in data[1]) # immutable
 225         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 226         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 227         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 228
 229     def failUnlessIsFooJSON(self, res):
 230         data = simplejson.loads(res)
 231         self.failUnless(isinstance(data, list))
 232         self.failUnlessEqual(data[0], "dirnode", res)
 233         self.failUnless(isinstance(data[1], dict))
 234         self.failUnless(data[1]["mutable"])
 235         self.failUnless("rw_uri" in data[1]) # mutable
 236         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 237         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 238         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 239
 240         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 241         self.failUnlessEqual(kidnames,
 242                              [u"bar.txt", u"blockingfile", u"empty",
 243                               u"n\u00fc.txt", u"sub"])
 244         kids = dict( [(unicode(name),value)
 245                       for (name,value)
 246                       in data[1]["children"].iteritems()] )
 247         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 248         self.failUnless("metadata" in kids[u"sub"][1])
 249         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 250         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 251         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 252         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 253         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 254         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 255                              self._bar_txt_verifycap)
 256         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 257                              self._bar_txt_metadata["ctime"])
 258         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 259                              self._bar_txt_uri)
 260
 261     def GET(self, urlpath, followRedirect=False, return_response=False,
 262             **kwargs):
 263         # if return_response=True, this fires with (data, statuscode,
 264         # respheaders) instead of just data.
 265         assert not isinstance(urlpath, unicode)
 266         url = self.webish_url + urlpath
 267         factory = HTTPClientGETFactory(url, method="GET",
 268                                        followRedirect=followRedirect, **kwargs)
 269         reactor.connectTCP("localhost", self.webish_port, factory)
 270         d = factory.deferred
 271         def _got_data(data):
 272             return (data, factory.status, factory.response_headers)
 273         if return_response:
 274             d.addCallback(_got_data)
 275         return factory.deferred
 276
 277     def HEAD(self, urlpath, return_response=False, **kwargs):
 278         # this requires some surgery, because twisted.web.client doesn't want
 279         # to give us back the response headers.
 280         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 281         reactor.connectTCP("localhost", self.webish_port, factory)
 282         d = factory.deferred
 283         def _got_data(data):
 284             return (data, factory.status, factory.response_headers)
 285         if return_response:
 286             d.addCallback(_got_data)
 287         return factory.deferred
 288
 289     def PUT(self, urlpath, data, **kwargs):
 290         url = self.webish_url + urlpath
 291         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 292
 293     def DELETE(self, urlpath):
 294         url = self.webish_url + urlpath
 295         return client.getPage(url, method="DELETE")
 296
 297     def POST(self, urlpath, followRedirect=False, **fields):
 298         url = self.webish_url + urlpath
 299         sepbase = "boogabooga"
 300         sep = "--" + sepbase
 301         form = []
 302         form.append(sep)
 303         form.append('Content-Disposition: form-data; name="_charset"')
 304         form.append('')
 305         form.append('UTF-8')
 306         form.append(sep)
 307         for name, value in fields.iteritems():
 308             if isinstance(value, tuple):
 309                 filename, value = value
 310                 form.append('Content-Disposition: form-data; name="%s"; '
 311                             'filename="%s"' % (name, filename.encode("utf-8")))
 312             else:
 313                 form.append('Content-Disposition: form-data; name="%s"' % name)
 314             form.append('')
 315             if isinstance(value, unicode):
 316                 value = value.encode("utf-8")
 317             else:
 318                 value = str(value)
 319             assert isinstance(value, str)
 320             form.append(value)
 321             form.append(sep)
 322         form[-1] += "--"
 323         body = "\r\n".join(form) + "\r\n"
 324         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 325                    }
 326         return client.getPage(url, method="POST", postdata=body,
 327                               headers=headers, followRedirect=followRedirect)
 328
 329     def shouldFail(self, res, expected_failure, which,
 330                    substring=None, response_substring=None):
 331         if isinstance(res, failure.Failure):
 332             res.trap(expected_failure)
 333             if substring:
 334                 self.failUnless(substring in str(res),
 335                                 "substring '%s' not in '%s'"
 336                                 % (substring, str(res)))
 337             if response_substring:
 338                 self.failUnless(response_substring in res.value.response,
 339                                 "response substring '%s' not in '%s'"
 340                                 % (response_substring, res.value.response))
 341         else:
 342             self.fail("%s was supposed to raise %s, not get '%s'" %
 343                       (which, expected_failure, res))
 344
 345     def shouldFail2(self, expected_failure, which, substring,
 346                     response_substring,
 347                     callable, *args, **kwargs):
 348         assert substring is None or isinstance(substring, str)
 349         assert response_substring is None or isinstance(response_substring, str)
 350         d = defer.maybeDeferred(callable, *args, **kwargs)
 351         def done(res):
 352             if isinstance(res, failure.Failure):
 353                 res.trap(expected_failure)
 354                 if substring:
 355                     self.failUnless(substring in str(res),
 356                                     "%s: substring '%s' not in '%s'"
 357                                     % (which, substring, str(res)))
 358                 if response_substring:
 359                     self.failUnless(response_substring in res.value.response,
 360                                     "%s: response substring '%s' not in '%s'"
 361                                     % (which,
 362                                        response_substring, res.value.response))
 363             else:
 364                 self.fail("%s was supposed to raise %s, not get '%s'" %
 365                           (which, expected_failure, res))
 366         d.addBoth(done)
 367         return d
 368
 369     def should404(self, res, which):
 370         if isinstance(res, failure.Failure):
 371             res.trap(error.Error)
 372             self.failUnlessEqual(res.value.status, "404")
 373         else:
 374             self.fail("%s was supposed to Error(404), not get '%s'" %
 375                       (which, res))
 376
 377     def shouldHTTPError(self, res, which, code=None, substring=None,
 378                         response_substring=None):
 379         if isinstance(res, failure.Failure):
 380             res.trap(error.Error)
 381             if code is not None:
 382                 self.failUnlessEqual(res.value.status, str(code))
 383             if substring:
 384                 self.failUnless(substring in str(res),
 385                                 "substring '%s' not in '%s'"
 386                                 % (substring, str(res)))
 387             if response_substring:
 388                 self.failUnless(response_substring in res.value.response,
 389                                 "response substring '%s' not in '%s'"
 390                                 % (response_substring, res.value.response))
 391         else:
 392             self.fail("%s was supposed to Error(%s), not get '%s'" %
 393                       (which, code, res))
 394
 395     def shouldHTTPError2(self, which,
 396                          code=None, substring=None, response_substring=None,
 397                          callable=None, *args, **kwargs):
 398         assert substring is None or isinstance(substring, str)
 399         assert callable
 400         d = defer.maybeDeferred(callable, *args, **kwargs)
 401         d.addBoth(self.shouldHTTPError, which,
 402                   code, substring, response_substring)
 403         return d
 404
 405
 406 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 407     def test_create(self):
 408         pass
 409
 410     def test_welcome(self):
 411         d = self.GET("/")
 412         def _check(res):
 413             self.failUnless('Welcome To AllMyData' in res)
 414             self.failUnless('Tahoe' in res)
 415
 416             self.s.basedir = 'web/test_welcome'
 417             fileutil.make_dirs("web/test_welcome")
 418             fileutil.make_dirs("web/test_welcome/private")
 419             return self.GET("/")
 420         d.addCallback(_check)
 421         return d
 422
 423     def test_provisioning(self):
 424         d = self.GET("/provisioning/")
 425         def _check(res):
 426             self.failUnless('Tahoe Provisioning Tool' in res)
 427             fields = {'filled': True,
 428                       "num_users": int(50e3),
 429                       "files_per_user": 1000,
 430                       "space_per_user": int(1e9),
 431                       "sharing_ratio": 1.0,
 432                       "encoding_parameters": "3-of-10-5",
 433                       "num_servers": 30,
 434                       "ownership_mode": "A",
 435                       "download_rate": 100,
 436                       "upload_rate": 10,
 437                       "delete_rate": 10,
 438                       "lease_timer": 7,
 439                       }
 440             return self.POST("/provisioning/", **fields)
 441
 442         d.addCallback(_check)
 443         def _check2(res):
 444             self.failUnless('Tahoe Provisioning Tool' in res)
 445             self.failUnless("Share space consumed: 167.01TB" in res)
 446
 447             fields = {'filled': True,
 448                       "num_users": int(50e6),
 449                       "files_per_user": 1000,
 450                       "space_per_user": int(5e9),
 451                       "sharing_ratio": 1.0,
 452                       "encoding_parameters": "25-of-100-50",
 453                       "num_servers": 30000,
 454                       "ownership_mode": "E",
 455                       "drive_failure_model": "U",
 456                       "drive_size": 1000,
 457                       "download_rate": 1000,
 458                       "upload_rate": 100,
 459                       "delete_rate": 100,
 460                       "lease_timer": 7,
 461                       }
 462             return self.POST("/provisioning/", **fields)
 463         d.addCallback(_check2)
 464         def _check3(res):
 465             self.failUnless("Share space consumed: huge!" in res)
 466             fields = {'filled': True}
 467             return self.POST("/provisioning/", **fields)
 468         d.addCallback(_check3)
 469         def _check4(res):
 470             self.failUnless("Share space consumed:" in res)
 471         d.addCallback(_check4)
 472         return d
 473
 474     def test_reliability_tool(self):
 475         try:
 476             from allmydata import reliability
 477             _hush_pyflakes = reliability
 478         except:
 479             raise unittest.SkipTest("reliability tool requires Numeric")
 480
 481         d = self.GET("/reliability/")
 482         def _check(res):
 483             self.failUnless('Tahoe Reliability Tool' in res)
 484             fields = {'drive_lifetime': "8Y",
 485                       "k": "3",
 486                       "R": "7",
 487                       "N": "10",
 488                       "delta": "100000",
 489                       "check_period": "1M",
 490                       "report_period": "3M",
 491                       "report_span": "5Y",
 492                       }
 493             return self.POST("/reliability/", **fields)
 494
 495         d.addCallback(_check)
 496         def _check2(res):
 497             self.failUnless('Tahoe Reliability Tool' in res)
 498             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 499             self.failUnless(re.search(r, res), res)
 500         d.addCallback(_check2)
 501         return d
 502
 503     def test_status(self):
 504         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 505         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 506         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 507         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 508         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 509         d = self.GET("/status", followRedirect=True)
 510         def _check(res):
 511             self.failUnless('Upload and Download Status' in res, res)
 512             self.failUnless('"down-%d"' % dl_num in res, res)
 513             self.failUnless('"up-%d"' % ul_num in res, res)
 514             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 515             self.failUnless('"publish-%d"' % pub_num in res, res)
 516             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 517         d.addCallback(_check)
 518         d.addCallback(lambda res: self.GET("/status/?t=json"))
 519         def _check_json(res):
 520             data = simplejson.loads(res)
 521             self.failUnless(isinstance(data, dict))
 522             active = data["active"]
 523             # TODO: test more. We need a way to fake an active operation
 524             # here.
 525         d.addCallback(_check_json)
 526
 527         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 528         def _check_dl(res):
 529             self.failUnless("File Download Status" in res, res)
 530         d.addCallback(_check_dl)
 531         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 532         def _check_ul(res):
 533             self.failUnless("File Upload Status" in res, res)
 534         d.addCallback(_check_ul)
 535         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 536         def _check_mapupdate(res):
 537             self.failUnless("Mutable File Servermap Update Status" in res, res)
 538         d.addCallback(_check_mapupdate)
 539         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 540         def _check_publish(res):
 541             self.failUnless("Mutable File Publish Status" in res, res)
 542         d.addCallback(_check_publish)
 543         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 544         def _check_retrieve(res):
 545             self.failUnless("Mutable File Retrieve Status" in res, res)
 546         d.addCallback(_check_retrieve)
 547
 548         return d
 549
 550     def test_status_numbers(self):
 551         drrm = status.DownloadResultsRendererMixin()
 552         self.failUnlessEqual(drrm.render_time(None, None), "")
 553         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 554         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 555         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 556         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 557         self.failUnlessEqual(drrm.render_rate(None, None), "")
 558         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 559         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 560         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 561
 562         urrm = status.UploadResultsRendererMixin()
 563         self.failUnlessEqual(urrm.render_time(None, None), "")
 564         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 565         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 566         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 567         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 568         self.failUnlessEqual(urrm.render_rate(None, None), "")
 569         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 570         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 571         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 572
 573     def test_GET_FILEURL(self):
 574         d = self.GET(self.public_url + "/foo/bar.txt")
 575         d.addCallback(self.failUnlessIsBarDotTxt)
 576         return d
 577
 578     def test_GET_FILEURL_range(self):
 579         headers = {"range": "bytes=1-10"}
 580         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 581                      return_response=True)
 582         def _got((res, status, headers)):
 583             self.failUnlessEqual(int(status), 206)
 584             self.failUnless(headers.has_key("content-range"))
 585             self.failUnlessEqual(headers["content-range"][0],
 586                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 587             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 588         d.addCallback(_got)
 589         return d
 590
 591     def test_GET_FILEURL_partial_range(self):
 592         headers = {"range": "bytes=5-"}
 593         length  = len(self.BAR_CONTENTS)
 594         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 595                      return_response=True)
 596         def _got((res, status, headers)):
 597             self.failUnlessEqual(int(status), 206)
 598             self.failUnless(headers.has_key("content-range"))
 599             self.failUnlessEqual(headers["content-range"][0],
 600                                  "bytes 5-%d/%d" % (length-1, length))
 601             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 602         d.addCallback(_got)
 603         return d
 604
 605     def test_HEAD_FILEURL_range(self):
 606         headers = {"range": "bytes=1-10"}
 607         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 608                      return_response=True)
 609         def _got((res, status, headers)):
 610             self.failUnlessEqual(res, "")
 611             self.failUnlessEqual(int(status), 206)
 612             self.failUnless(headers.has_key("content-range"))
 613             self.failUnlessEqual(headers["content-range"][0],
 614                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 615         d.addCallback(_got)
 616         return d
 617
 618     def test_HEAD_FILEURL_partial_range(self):
 619         headers = {"range": "bytes=5-"}
 620         length  = len(self.BAR_CONTENTS)
 621         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 622                      return_response=True)
 623         def _got((res, status, headers)):
 624             self.failUnlessEqual(int(status), 206)
 625             self.failUnless(headers.has_key("content-range"))
 626             self.failUnlessEqual(headers["content-range"][0],
 627                                  "bytes 5-%d/%d" % (length-1, length))
 628         d.addCallback(_got)
 629         return d
 630
 631     def test_GET_FILEURL_range_bad(self):
 632         headers = {"range": "BOGUS=fizbop-quarnak"}
 633         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 634                              "400 Bad Request",
 635                              "Syntactically invalid http range header",
 636                              self.GET, self.public_url + "/foo/bar.txt",
 637                              headers=headers)
 638         return d
 639
 640     def test_HEAD_FILEURL(self):
 641         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 642         def _got((res, status, headers)):
 643             self.failUnlessEqual(res, "")
 644             self.failUnlessEqual(headers["content-length"][0],
 645                                  str(len(self.BAR_CONTENTS)))
 646             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_named(self):
 651         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 652         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 653         d = self.GET(base + "/@@name=/blah.txt")
 654         d.addCallback(self.failUnlessIsBarDotTxt)
 655         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 656         d.addCallback(self.failUnlessIsBarDotTxt)
 657         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 658         d.addCallback(self.failUnlessIsBarDotTxt)
 659         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 660         d.addCallback(self.failUnlessIsBarDotTxt)
 661         save_url = base + "?save=true&filename=blah.txt"
 662         d.addCallback(lambda res: self.GET(save_url))
 663         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 664         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 665         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 666         u_url = base + "?save=true&filename=" + u_fn_e
 667         d.addCallback(lambda res: self.GET(u_url))
 668         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 669         return d
 670
 671     def test_PUT_FILEURL_named_bad(self):
 672         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 673         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 674                              "400 Bad Request",
 675                              "/file can only be used with GET or HEAD",
 676                              self.PUT, base + "/@@name=/blah.txt", "")
 677         return d
 678
 679     def test_GET_DIRURL_named_bad(self):
 680         base = "/file/%s" % urllib.quote(self._foo_uri)
 681         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 682                              "400 Bad Request",
 683                              "is not a file-cap",
 684                              self.GET, base + "/@@name=/blah.txt")
 685         return d
 686
 687     def test_GET_slash_file_bad(self):
 688         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 689                              "404 Not Found",
 690                              "/file must be followed by a file-cap and a name",
 691                              self.GET, "/file")
 692         return d
 693
 694     def test_GET_unhandled_URI_named(self):
 695         contents, n, newuri = self.makefile(12)
 696         verifier_cap = n.get_verify_cap().to_string()
 697         base = "/file/%s" % urllib.quote(verifier_cap)
 698         # client.create_node_from_uri() can't handle verify-caps
 699         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 700                              "400 Bad Request",
 701                              "is not a valid file- or directory- cap",
 702                              self.GET, base)
 703         return d
 704
 705     def test_GET_unhandled_URI(self):
 706         contents, n, newuri = self.makefile(12)
 707         verifier_cap = n.get_verify_cap().to_string()
 708         base = "/uri/%s" % urllib.quote(verifier_cap)
 709         # client.create_node_from_uri() can't handle verify-caps
 710         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 711                              "400 Bad Request",
 712                              "is not a valid file- or directory- cap",
 713                              self.GET, base)
 714         return d
 715
 716     def test_GET_FILE_URI(self):
 717         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 718         d = self.GET(base)
 719         d.addCallback(self.failUnlessIsBarDotTxt)
 720         return d
 721
 722     def test_GET_FILE_URI_badchild(self):
 723         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 724         errmsg = "Files have no children, certainly not named 'boguschild'"
 725         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 726                              "400 Bad Request", errmsg,
 727                              self.GET, base)
 728         return d
 729
 730     def test_PUT_FILE_URI_badchild(self):
 731         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 732         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 733         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 734                              "400 Bad Request", errmsg,
 735                              self.PUT, base, "")
 736         return d
 737
 738     def test_GET_FILEURL_save(self):
 739         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 740         # TODO: look at the headers, expect a Content-Disposition: attachment
 741         # header.
 742         d.addCallback(self.failUnlessIsBarDotTxt)
 743         return d
 744
 745     def test_GET_FILEURL_missing(self):
 746         d = self.GET(self.public_url + "/foo/missing")
 747         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 748         return d
 749
 750     def test_PUT_NEWFILEURL(self):
 751         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 752         # TODO: we lose the response code, so we can't check this
 753         #self.failUnlessEqual(responsecode, 201)
 754         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 755         d.addCallback(lambda res:
 756                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 757                                                       self.NEWFILE_CONTENTS))
 758         return d
 759
 760     def test_PUT_NEWFILEURL_range_bad(self):
 761         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 762         target = self.public_url + "/foo/new.txt"
 763         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 764                              "501 Not Implemented",
 765                              "Content-Range in PUT not yet supported",
 766                              # (and certainly not for immutable files)
 767                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 768                              headers=headers)
 769         d.addCallback(lambda res:
 770                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 771         return d
 772
 773     def test_PUT_NEWFILEURL_mutable(self):
 774         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 775                      self.NEWFILE_CONTENTS)
 776         # TODO: we lose the response code, so we can't check this
 777         #self.failUnlessEqual(responsecode, 201)
 778         def _check_uri(res):
 779             u = uri.from_string_mutable_filenode(res)
 780             self.failUnless(u.is_mutable())
 781             self.failIf(u.is_readonly())
 782             return res
 783         d.addCallback(_check_uri)
 784         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 785         d.addCallback(lambda res:
 786                       self.failUnlessMutableChildContentsAre(self._foo_node,
 787                                                              u"new.txt",
 788                                                              self.NEWFILE_CONTENTS))
 789         return d
 790
 791     def test_PUT_NEWFILEURL_mutable_toobig(self):
 792         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 793                              "413 Request Entity Too Large",
 794                              "SDMF is limited to one segment, and 10001 > 10000",
 795                              self.PUT,
 796                              self.public_url + "/foo/new.txt?mutable=true",
 797                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 798         return d
 799
 800     def test_PUT_NEWFILEURL_replace(self):
 801         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 802         # TODO: we lose the response code, so we can't check this
 803         #self.failUnlessEqual(responsecode, 200)
 804         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 805         d.addCallback(lambda res:
 806                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 807                                                       self.NEWFILE_CONTENTS))
 808         return d
 809
 810     def test_PUT_NEWFILEURL_bad_t(self):
 811         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 812                              "PUT to a file: bad t=bogus",
 813                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 814                              "contents")
 815         return d
 816
 817     def test_PUT_NEWFILEURL_no_replace(self):
 818         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 819                      self.NEWFILE_CONTENTS)
 820         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 821                   "409 Conflict",
 822                   "There was already a child by that name, and you asked me "
 823                   "to not replace it")
 824         return d
 825
 826     def test_PUT_NEWFILEURL_mkdirs(self):
 827         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 828         fn = self._foo_node
 829         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 830         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 831         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 832         d.addCallback(lambda res:
 833                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 834                                                       self.NEWFILE_CONTENTS))
 835         return d
 836
 837     def test_PUT_NEWFILEURL_blocked(self):
 838         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 839                      self.NEWFILE_CONTENTS)
 840         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 841                   "409 Conflict",
 842                   "Unable to create directory 'blockingfile': a file was in the way")
 843         return d
 844
 845     def test_DELETE_FILEURL(self):
 846         d = self.DELETE(self.public_url + "/foo/bar.txt")
 847         d.addCallback(lambda res:
 848                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 849         return d
 850
 851     def test_DELETE_FILEURL_missing(self):
 852         d = self.DELETE(self.public_url + "/foo/missing")
 853         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 854         return d
 855
 856     def test_DELETE_FILEURL_missing2(self):
 857         d = self.DELETE(self.public_url + "/missing/missing")
 858         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 859         return d
 860
 861     def test_GET_FILEURL_json(self):
 862         # twisted.web.http.parse_qs ignores any query args without an '=', so
 863         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 864         # instead. This may make it tricky to emulate the S3 interface
 865         # completely.
 866         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 867         d.addCallback(self.failUnlessIsBarJSON)
 868         return d
 869
 870     def test_GET_FILEURL_json_missing(self):
 871         d = self.GET(self.public_url + "/foo/missing?json")
 872         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 873         return d
 874
 875     def test_GET_FILEURL_uri(self):
 876         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 877         def _check(res):
 878             self.failUnlessEqual(res, self._bar_txt_uri)
 879         d.addCallback(_check)
 880         d.addCallback(lambda res:
 881                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 882         def _check2(res):
 883             # for now, for files, uris and readonly-uris are the same
 884             self.failUnlessEqual(res, self._bar_txt_uri)
 885         d.addCallback(_check2)
 886         return d
 887
 888     def test_GET_FILEURL_badtype(self):
 889         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 890                                   "bad t=bogus",
 891                                   self.GET,
 892                                   self.public_url + "/foo/bar.txt?t=bogus")
 893         return d
 894
 895     def test_GET_FILEURL_uri_missing(self):
 896         d = self.GET(self.public_url + "/foo/missing?t=uri")
 897         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 898         return d
 899
 900     def test_GET_DIRURL(self):
 901         # the addSlash means we get a redirect here
 902         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 903         ROOT = "../../.."
 904         d = self.GET(self.public_url + "/foo", followRedirect=True)
 905         def _check(res):
 906             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 907                             in res, res)
 908             # the FILE reference points to a URI, but it should end in bar.txt
 909             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 910                        (ROOT, urllib.quote(self._bar_txt_uri)))
 911             get_bar = "".join([r'<td>',
 912                                r'<a href="%s">bar.txt</a>' % bar_url,
 913                                r'</td>',
 914                                r'\s+<td>FILE</td>',
 915                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 916                                ])
 917             self.failUnless(re.search(get_bar, res), res)
 918             for line in res.split("\n"):
 919                 # find the line that contains the delete button for bar.txt
 920                 if ("form action" in line and
 921                     'value="delete"' in line and
 922                     'value="bar.txt"' in line):
 923                     # the form target should use a relative URL
 924                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 925                     self.failUnless(('action="%s"' % foo_url) in line, line)
 926                     # and the when_done= should too
 927                     #done_url = urllib.quote(???)
 928                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 929                     #                in line, line)
 930                     break
 931             else:
 932                 self.fail("unable to find delete-bar.txt line", res)
 933
 934             # the DIR reference just points to a URI
 935             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 936             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 937                        + r'\s+<td>DIR</td>')
 938             self.failUnless(re.search(get_sub, res), res)
 939         d.addCallback(_check)
 940
 941         # look at a directory which is readonly
 942         d.addCallback(lambda res:
 943                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 944         def _check2(res):
 945             self.failUnless("(readonly)" in res, res)
 946             self.failIf("Upload a file" in res, res)
 947         d.addCallback(_check2)
 948
 949         # and at a directory that contains a readonly directory
 950         d.addCallback(lambda res:
 951                       self.GET(self.public_url, followRedirect=True))
 952         def _check3(res):
 953             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 954                                       '</td>\s+<td>DIR-RO</td>', res))
 955         d.addCallback(_check3)
 956
 957         return d
 958
 959     def test_GET_DIRURL_badtype(self):
 960         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 961                                   400, "Bad Request",
 962                                   "bad t=bogus",
 963                                   self.GET,
 964                                   self.public_url + "/foo?t=bogus")
 965         return d
 966
 967     def test_GET_DIRURL_json(self):
 968         d = self.GET(self.public_url + "/foo?t=json")
 969         d.addCallback(self.failUnlessIsFooJSON)
 970         return d
 971
 972
 973     def test_POST_DIRURL_manifest_no_ophandle(self):
 974         d = self.shouldFail2(error.Error,
 975                              "test_POST_DIRURL_manifest_no_ophandle",
 976                              "400 Bad Request",
 977                              "slow operation requires ophandle=",
 978                              self.POST, self.public_url, t="start-manifest")
 979         return d
 980
 981     def test_POST_DIRURL_manifest(self):
 982         d = defer.succeed(None)
 983         def getman(ignored, output):
 984             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 985                           followRedirect=True)
 986             d.addCallback(self.wait_for_operation, "125")
 987             d.addCallback(self.get_operation_results, "125", output)
 988             return d
 989         d.addCallback(getman, None)
 990         def _got_html(manifest):
 991             self.failUnless("Manifest of SI=" in manifest)
 992             self.failUnless("<td>sub</td>" in manifest)
 993             self.failUnless(self._sub_uri in manifest)
 994             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 995         d.addCallback(_got_html)
 996
 997         # both t=status and unadorned GET should be identical
 998         d.addCallback(lambda res: self.GET("/operations/125"))
 999         d.addCallback(_got_html)
1000
1001         d.addCallback(getman, "html")
1002         d.addCallback(_got_html)
1003         d.addCallback(getman, "text")
1004         def _got_text(manifest):
1005             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1006             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1007         d.addCallback(_got_text)
1008         d.addCallback(getman, "JSON")
1009         def _got_json(res):
1010             data = res["manifest"]
1011             got = {}
1012             for (path_list, cap) in data:
1013                 got[tuple(path_list)] = cap
1014             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1015             self.failUnless((u"sub",u"baz.txt") in got)
1016             self.failUnless("finished" in res)
1017             self.failUnless("origin" in res)
1018             self.failUnless("storage-index" in res)
1019             self.failUnless("verifycaps" in res)
1020             self.failUnless("stats" in res)
1021         d.addCallback(_got_json)
1022         return d
1023
1024     def test_POST_DIRURL_deepsize_no_ophandle(self):
1025         d = self.shouldFail2(error.Error,
1026                              "test_POST_DIRURL_deepsize_no_ophandle",
1027                              "400 Bad Request",
1028                              "slow operation requires ophandle=",
1029                              self.POST, self.public_url, t="start-deep-size")
1030         return d
1031
1032     def test_POST_DIRURL_deepsize(self):
1033         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1034                       followRedirect=True)
1035         d.addCallback(self.wait_for_operation, "126")
1036         d.addCallback(self.get_operation_results, "126", "json")
1037         def _got_json(data):
1038             self.failUnlessEqual(data["finished"], True)
1039             size = data["size"]
1040             self.failUnless(size > 1000)
1041         d.addCallback(_got_json)
1042         d.addCallback(self.get_operation_results, "126", "text")
1043         def _got_text(res):
1044             mo = re.search(r'^size: (\d+)$', res, re.M)
1045             self.failUnless(mo, res)
1046             size = int(mo.group(1))
1047             # with directories, the size varies.
1048             self.failUnless(size > 1000)
1049         d.addCallback(_got_text)
1050         return d
1051
1052     def test_POST_DIRURL_deepstats_no_ophandle(self):
1053         d = self.shouldFail2(error.Error,
1054                              "test_POST_DIRURL_deepstats_no_ophandle",
1055                              "400 Bad Request",
1056                              "slow operation requires ophandle=",
1057                              self.POST, self.public_url, t="start-deep-stats")
1058         return d
1059
1060     def test_POST_DIRURL_deepstats(self):
1061         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1062                       followRedirect=True)
1063         d.addCallback(self.wait_for_operation, "127")
1064         d.addCallback(self.get_operation_results, "127", "json")
1065         def _got_json(stats):
1066             expected = {"count-immutable-files": 3,
1067                         "count-mutable-files": 0,
1068                         "count-literal-files": 0,
1069                         "count-files": 3,
1070                         "count-directories": 3,
1071                         "size-immutable-files": 57,
1072                         "size-literal-files": 0,
1073                         #"size-directories": 1912, # varies
1074                         #"largest-directory": 1590,
1075                         "largest-directory-children": 5,
1076                         "largest-immutable-file": 19,
1077                         }
1078             for k,v in expected.iteritems():
1079                 self.failUnlessEqual(stats[k], v,
1080                                      "stats[%s] was %s, not %s" %
1081                                      (k, stats[k], v))
1082             self.failUnlessEqual(stats["size-files-histogram"],
1083                                  [ [11, 31, 3] ])
1084         d.addCallback(_got_json)
1085         return d
1086
1087     def test_POST_DIRURL_stream_manifest(self):
1088         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1089         def _check(res):
1090             self.failUnless(res.endswith("\n"))
1091             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1092             self.failUnlessEqual(len(units), 7)
1093             self.failUnlessEqual(units[-1]["type"], "stats")
1094             first = units[0]
1095             self.failUnlessEqual(first["path"], [])
1096             self.failUnlessEqual(first["cap"], self._foo_uri)
1097             self.failUnlessEqual(first["type"], "directory")
1098             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1099             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1100             self.failIfEqual(baz["storage-index"], None)
1101             self.failIfEqual(baz["verifycap"], None)
1102             self.failIfEqual(baz["repaircap"], None)
1103             return
1104         d.addCallback(_check)
1105         return d
1106
1107     def test_GET_DIRURL_uri(self):
1108         d = self.GET(self.public_url + "/foo?t=uri")
1109         def _check(res):
1110             self.failUnlessEqual(res, self._foo_uri)
1111         d.addCallback(_check)
1112         return d
1113
1114     def test_GET_DIRURL_readonly_uri(self):
1115         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1116         def _check(res):
1117             self.failUnlessEqual(res, self._foo_readonly_uri)
1118         d.addCallback(_check)
1119         return d
1120
1121     def test_PUT_NEWDIRURL(self):
1122         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1123         d.addCallback(lambda res:
1124                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1125         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1126         d.addCallback(self.failUnlessNodeKeysAre, [])
1127         return d
1128
1129     def test_PUT_NEWDIRURL_exists(self):
1130         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1131         d.addCallback(lambda res:
1132                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1133         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1134         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1135         return d
1136
1137     def test_PUT_NEWDIRURL_blocked(self):
1138         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1139                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1140                              self.PUT,
1141                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1142         d.addCallback(lambda res:
1143                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1144         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1145         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1146         return d
1147
1148     def test_PUT_NEWDIRURL_mkdir_p(self):
1149         d = defer.succeed(None)
1150         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1151         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1152         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1153         def mkdir_p(mkpnode):
1154             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1155             d = self.POST(url)
1156             def made_subsub(ssuri):
1157                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1158                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1159                 d = self.POST(url)
1160                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1161                 return d
1162             d.addCallback(made_subsub)
1163             return d
1164         d.addCallback(mkdir_p)
1165         return d
1166
1167     def test_PUT_NEWDIRURL_mkdirs(self):
1168         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1169         d.addCallback(lambda res:
1170                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1171         d.addCallback(lambda res:
1172                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1173         d.addCallback(lambda res:
1174                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1175         d.addCallback(self.failUnlessNodeKeysAre, [])
1176         return d
1177
1178     def test_DELETE_DIRURL(self):
1179         d = self.DELETE(self.public_url + "/foo")
1180         d.addCallback(lambda res:
1181                       self.failIfNodeHasChild(self.public_root, u"foo"))
1182         return d
1183
1184     def test_DELETE_DIRURL_missing(self):
1185         d = self.DELETE(self.public_url + "/foo/missing")
1186         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1187         d.addCallback(lambda res:
1188                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1189         return d
1190
1191     def test_DELETE_DIRURL_missing2(self):
1192         d = self.DELETE(self.public_url + "/missing")
1193         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1194         return d
1195
1196     def dump_root(self):
1197         print "NODEWALK"
1198         w = webish.DirnodeWalkerMixin()
1199         def visitor(childpath, childnode, metadata):
1200             print childpath
1201         d = w.walk(self.public_root, visitor)
1202         return d
1203
1204     def failUnlessNodeKeysAre(self, node, expected_keys):
1205         for k in expected_keys:
1206             assert isinstance(k, unicode)
1207         d = node.list()
1208         def _check(children):
1209             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1210         d.addCallback(_check)
1211         return d
1212     def failUnlessNodeHasChild(self, node, name):
1213         assert isinstance(name, unicode)
1214         d = node.list()
1215         def _check(children):
1216             self.failUnless(name in children)
1217         d.addCallback(_check)
1218         return d
1219     def failIfNodeHasChild(self, node, name):
1220         assert isinstance(name, unicode)
1221         d = node.list()
1222         def _check(children):
1223             self.failIf(name in children)
1224         d.addCallback(_check)
1225         return d
1226
1227     def failUnlessChildContentsAre(self, node, name, expected_contents):
1228         assert isinstance(name, unicode)
1229         d = node.get_child_at_path(name)
1230         d.addCallback(lambda node: node.download_to_data())
1231         def _check(contents):
1232             self.failUnlessEqual(contents, expected_contents)
1233         d.addCallback(_check)
1234         return d
1235
1236     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1237         assert isinstance(name, unicode)
1238         d = node.get_child_at_path(name)
1239         d.addCallback(lambda node: node.download_best_version())
1240         def _check(contents):
1241             self.failUnlessEqual(contents, expected_contents)
1242         d.addCallback(_check)
1243         return d
1244
1245     def failUnlessChildURIIs(self, node, name, expected_uri):
1246         assert isinstance(name, unicode)
1247         d = node.get_child_at_path(name)
1248         def _check(child):
1249             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1250         d.addCallback(_check)
1251         return d
1252
1253     def failUnlessURIMatchesChild(self, got_uri, node, name):
1254         assert isinstance(name, unicode)
1255         d = node.get_child_at_path(name)
1256         def _check(child):
1257             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1258         d.addCallback(_check)
1259         return d
1260
1261     def failUnlessCHKURIHasContents(self, got_uri, contents):
1262         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1263
1264     def test_POST_upload(self):
1265         d = self.POST(self.public_url + "/foo", t="upload",
1266                       file=("new.txt", self.NEWFILE_CONTENTS))
1267         fn = self._foo_node
1268         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1269         d.addCallback(lambda res:
1270                       self.failUnlessChildContentsAre(fn, u"new.txt",
1271                                                       self.NEWFILE_CONTENTS))
1272         return d
1273
1274     def test_POST_upload_unicode(self):
1275         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1276         d = self.POST(self.public_url + "/foo", t="upload",
1277                       file=(filename, self.NEWFILE_CONTENTS))
1278         fn = self._foo_node
1279         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1280         d.addCallback(lambda res:
1281                       self.failUnlessChildContentsAre(fn, filename,
1282                                                       self.NEWFILE_CONTENTS))
1283         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1284         d.addCallback(lambda res: self.GET(target_url))
1285         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1286                                                             self.NEWFILE_CONTENTS,
1287                                                             contents))
1288         return d
1289
1290     def test_POST_upload_unicode_named(self):
1291         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1292         d = self.POST(self.public_url + "/foo", t="upload",
1293                       name=filename,
1294                       file=("overridden", self.NEWFILE_CONTENTS))
1295         fn = self._foo_node
1296         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1297         d.addCallback(lambda res:
1298                       self.failUnlessChildContentsAre(fn, filename,
1299                                                       self.NEWFILE_CONTENTS))
1300         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1301         d.addCallback(lambda res: self.GET(target_url))
1302         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1303                                                             self.NEWFILE_CONTENTS,
1304                                                             contents))
1305         return d
1306
1307     def test_POST_upload_no_link(self):
1308         d = self.POST("/uri", t="upload",
1309                       file=("new.txt", self.NEWFILE_CONTENTS))
1310         def _check_upload_results(page):
1311             # this should be a page which describes the results of the upload
1312             # that just finished.
1313             self.failUnless("Upload Results:" in page)
1314             self.failUnless("URI:" in page)
1315             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1316             mo = uri_re.search(page)
1317             self.failUnless(mo, page)
1318             new_uri = mo.group(1)
1319             return new_uri
1320         d.addCallback(_check_upload_results)
1321         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1322         return d
1323
1324     def test_POST_upload_no_link_whendone(self):
1325         d = self.POST("/uri", t="upload", when_done="/",
1326                       file=("new.txt", self.NEWFILE_CONTENTS))
1327         d.addBoth(self.shouldRedirect, "/")
1328         return d
1329
1330     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1331         d = defer.maybeDeferred(callable, *args, **kwargs)
1332         def done(res):
1333             if isinstance(res, failure.Failure):
1334                 res.trap(error.PageRedirect)
1335                 statuscode = res.value.status
1336                 target = res.value.location
1337                 return checker(statuscode, target)
1338             self.fail("%s: callable was supposed to redirect, not return '%s'"
1339                       % (which, res))
1340         d.addBoth(done)
1341         return d
1342
1343     def test_POST_upload_no_link_whendone_results(self):
1344         def check(statuscode, target):
1345             self.failUnlessEqual(statuscode, str(http.FOUND))
1346             self.failUnless(target.startswith(self.webish_url), target)
1347             return client.getPage(target, method="GET")
1348         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1349                                  check,
1350                                  self.POST, "/uri", t="upload",
1351                                  when_done="/uri/%(uri)s",
1352                                  file=("new.txt", self.NEWFILE_CONTENTS))
1353         d.addCallback(lambda res:
1354                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1355         return d
1356
1357     def test_POST_upload_no_link_mutable(self):
1358         d = self.POST("/uri", t="upload", mutable="true",
1359                       file=("new.txt", self.NEWFILE_CONTENTS))
1360         def _check(new_uri):
1361             new_uri = new_uri.strip()
1362             self.new_uri = new_uri
1363             u = IURI(new_uri)
1364             self.failUnless(IMutableFileURI.providedBy(u))
1365             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1366             n = self.s.create_node_from_uri(new_uri)
1367             return n.download_best_version()
1368         d.addCallback(_check)
1369         def _check2(data):
1370             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1371             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1372         d.addCallback(_check2)
1373         def _check3(data):
1374             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1375             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1376         d.addCallback(_check3)
1377         def _check4(data):
1378             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1379         d.addCallback(_check4)
1380         return d
1381
1382     def test_POST_upload_no_link_mutable_toobig(self):
1383         d = self.shouldFail2(error.Error,
1384                              "test_POST_upload_no_link_mutable_toobig",
1385                              "413 Request Entity Too Large",
1386                              "SDMF is limited to one segment, and 10001 > 10000",
1387                              self.POST,
1388                              "/uri", t="upload", mutable="true",
1389                              file=("new.txt",
1390                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1391         return d
1392
1393     def test_POST_upload_mutable(self):
1394         # this creates a mutable file
1395         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1396                       file=("new.txt", self.NEWFILE_CONTENTS))
1397         fn = self._foo_node
1398         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1399         d.addCallback(lambda res:
1400                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1401                                                              self.NEWFILE_CONTENTS))
1402         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1403         def _got(newnode):
1404             self.failUnless(IMutableFileNode.providedBy(newnode))
1405             self.failUnless(newnode.is_mutable())
1406             self.failIf(newnode.is_readonly())
1407             self._mutable_node = newnode
1408             self._mutable_uri = newnode.get_uri()
1409         d.addCallback(_got)
1410
1411         # now upload it again and make sure that the URI doesn't change
1412         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1413         d.addCallback(lambda res:
1414                       self.POST(self.public_url + "/foo", t="upload",
1415                                 mutable="true",
1416                                 file=("new.txt", NEWER_CONTENTS)))
1417         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1418         d.addCallback(lambda res:
1419                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1420                                                              NEWER_CONTENTS))
1421         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1422         def _got2(newnode):
1423             self.failUnless(IMutableFileNode.providedBy(newnode))
1424             self.failUnless(newnode.is_mutable())
1425             self.failIf(newnode.is_readonly())
1426             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1427         d.addCallback(_got2)
1428
1429         # upload a second time, using PUT instead of POST
1430         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1431         d.addCallback(lambda res:
1432                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1433         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1434         d.addCallback(lambda res:
1435                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1436                                                              NEW2_CONTENTS))
1437
1438         # finally list the directory, since mutable files are displayed
1439         # slightly differently
1440
1441         d.addCallback(lambda res:
1442                       self.GET(self.public_url + "/foo/",
1443                                followRedirect=True))
1444         def _check_page(res):
1445             # TODO: assert more about the contents
1446             self.failUnless("SSK" in res)
1447             return res
1448         d.addCallback(_check_page)
1449
1450         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1451         def _got3(newnode):
1452             self.failUnless(IMutableFileNode.providedBy(newnode))
1453             self.failUnless(newnode.is_mutable())
1454             self.failIf(newnode.is_readonly())
1455             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1456         d.addCallback(_got3)
1457
1458         # look at the JSON form of the enclosing directory
1459         d.addCallback(lambda res:
1460                       self.GET(self.public_url + "/foo/?t=json",
1461                                followRedirect=True))
1462         def _check_page_json(res):
1463             parsed = simplejson.loads(res)
1464             self.failUnlessEqual(parsed[0], "dirnode")
1465             children = dict( [(unicode(name),value)
1466                               for (name,value)
1467                               in parsed[1]["children"].iteritems()] )
1468             self.failUnless("new.txt" in children)
1469             new_json = children["new.txt"]
1470             self.failUnlessEqual(new_json[0], "filenode")
1471             self.failUnless(new_json[1]["mutable"])
1472             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1473             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1474             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1475         d.addCallback(_check_page_json)
1476
1477         # and the JSON form of the file
1478         d.addCallback(lambda res:
1479                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1480         def _check_file_json(res):
1481             parsed = simplejson.loads(res)
1482             self.failUnlessEqual(parsed[0], "filenode")
1483             self.failUnless(parsed[1]["mutable"])
1484             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1485             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1486             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1487         d.addCallback(_check_file_json)
1488
1489         # and look at t=uri and t=readonly-uri
1490         d.addCallback(lambda res:
1491                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1492         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1493         d.addCallback(lambda res:
1494                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1495         def _check_ro_uri(res):
1496             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1497             self.failUnlessEqual(res, ro_uri)
1498         d.addCallback(_check_ro_uri)
1499
1500         # make sure we can get to it from /uri/URI
1501         d.addCallback(lambda res:
1502                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1503         d.addCallback(lambda res:
1504                       self.failUnlessEqual(res, NEW2_CONTENTS))
1505
1506         # and that HEAD computes the size correctly
1507         d.addCallback(lambda res:
1508                       self.HEAD(self.public_url + "/foo/new.txt",
1509                                 return_response=True))
1510         def _got_headers((res, status, headers)):
1511             self.failUnlessEqual(res, "")
1512             self.failUnlessEqual(headers["content-length"][0],
1513                                  str(len(NEW2_CONTENTS)))
1514             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1515         d.addCallback(_got_headers)
1516
1517         # make sure that size errors are displayed correctly for overwrite
1518         d.addCallback(lambda res:
1519                       self.shouldFail2(error.Error,
1520                                        "test_POST_upload_mutable-toobig",
1521                                        "413 Request Entity Too Large",
1522                                        "SDMF is limited to one segment, and 10001 > 10000",
1523                                        self.POST,
1524                                        self.public_url + "/foo", t="upload",
1525                                        mutable="true",
1526                                        file=("new.txt",
1527                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1528                                        ))
1529
1530         d.addErrback(self.dump_error)
1531         return d
1532
1533     def test_POST_upload_mutable_toobig(self):
1534         d = self.shouldFail2(error.Error,
1535                              "test_POST_upload_no_link_mutable_toobig",
1536                              "413 Request Entity Too Large",
1537                              "SDMF is limited to one segment, and 10001 > 10000",
1538                              self.POST,
1539                              self.public_url + "/foo",
1540                              t="upload", mutable="true",
1541                              file=("new.txt",
1542                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1543         return d
1544
1545     def dump_error(self, f):
1546         # if the web server returns an error code (like 400 Bad Request),
1547         # web.client.getPage puts the HTTP response body into the .response
1548         # attribute of the exception object that it gives back. It does not
1549         # appear in the Failure's repr(), so the ERROR that trial displays
1550         # will be rather terse and unhelpful. addErrback this method to the
1551         # end of your chain to get more information out of these errors.
1552         if f.check(error.Error):
1553             print "web.error.Error:"
1554             print f
1555             print f.value.response
1556         return f
1557
1558     def test_POST_upload_replace(self):
1559         d = self.POST(self.public_url + "/foo", t="upload",
1560                       file=("bar.txt", self.NEWFILE_CONTENTS))
1561         fn = self._foo_node
1562         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1563         d.addCallback(lambda res:
1564                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1565                                                       self.NEWFILE_CONTENTS))
1566         return d
1567
1568     def test_POST_upload_no_replace_ok(self):
1569         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1570                       file=("new.txt", self.NEWFILE_CONTENTS))
1571         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1572         d.addCallback(lambda res: self.failUnlessEqual(res,
1573                                                        self.NEWFILE_CONTENTS))
1574         return d
1575
1576     def test_POST_upload_no_replace_queryarg(self):
1577         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1578                       file=("bar.txt", self.NEWFILE_CONTENTS))
1579         d.addBoth(self.shouldFail, error.Error,
1580                   "POST_upload_no_replace_queryarg",
1581                   "409 Conflict",
1582                   "There was already a child by that name, and you asked me "
1583                   "to not replace it")
1584         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1585         d.addCallback(self.failUnlessIsBarDotTxt)
1586         return d
1587
1588     def test_POST_upload_no_replace_field(self):
1589         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1590                       file=("bar.txt", self.NEWFILE_CONTENTS))
1591         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1592                   "409 Conflict",
1593                   "There was already a child by that name, and you asked me "
1594                   "to not replace it")
1595         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1596         d.addCallback(self.failUnlessIsBarDotTxt)
1597         return d
1598
1599     def test_POST_upload_whendone(self):
1600         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1601                       file=("new.txt", self.NEWFILE_CONTENTS))
1602         d.addBoth(self.shouldRedirect, "/THERE")
1603         fn = self._foo_node
1604         d.addCallback(lambda res:
1605                       self.failUnlessChildContentsAre(fn, u"new.txt",
1606                                                       self.NEWFILE_CONTENTS))
1607         return d
1608
1609     def test_POST_upload_named(self):
1610         fn = self._foo_node
1611         d = self.POST(self.public_url + "/foo", t="upload",
1612                       name="new.txt", file=self.NEWFILE_CONTENTS)
1613         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1614         d.addCallback(lambda res:
1615                       self.failUnlessChildContentsAre(fn, u"new.txt",
1616                                                       self.NEWFILE_CONTENTS))
1617         return d
1618
1619     def test_POST_upload_named_badfilename(self):
1620         d = self.POST(self.public_url + "/foo", t="upload",
1621                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1622         d.addBoth(self.shouldFail, error.Error,
1623                   "test_POST_upload_named_badfilename",
1624                   "400 Bad Request",
1625                   "name= may not contain a slash",
1626                   )
1627         # make sure that nothing was added
1628         d.addCallback(lambda res:
1629                       self.failUnlessNodeKeysAre(self._foo_node,
1630                                                  [u"bar.txt", u"blockingfile",
1631                                                   u"empty", u"n\u00fc.txt",
1632                                                   u"sub"]))
1633         return d
1634
1635     def test_POST_FILEURL_check(self):
1636         bar_url = self.public_url + "/foo/bar.txt"
1637         d = self.POST(bar_url, t="check")
1638         def _check(res):
1639             self.failUnless("Healthy :" in res)
1640         d.addCallback(_check)
1641         redir_url = "http://allmydata.org/TARGET"
1642         def _check2(statuscode, target):
1643             self.failUnlessEqual(statuscode, str(http.FOUND))
1644             self.failUnlessEqual(target, redir_url)
1645         d.addCallback(lambda res:
1646                       self.shouldRedirect2("test_POST_FILEURL_check",
1647                                            _check2,
1648                                            self.POST, bar_url,
1649                                            t="check",
1650                                            when_done=redir_url))
1651         d.addCallback(lambda res:
1652                       self.POST(bar_url, t="check", return_to=redir_url))
1653         def _check3(res):
1654             self.failUnless("Healthy :" in res)
1655             self.failUnless("Return to parent directory" in res)
1656             self.failUnless(redir_url in res)
1657         d.addCallback(_check3)
1658
1659         d.addCallback(lambda res:
1660                       self.POST(bar_url, t="check", output="JSON"))
1661         def _check_json(res):
1662             data = simplejson.loads(res)
1663             self.failUnless("storage-index" in data)
1664             self.failUnless(data["results"]["healthy"])
1665         d.addCallback(_check_json)
1666
1667         return d
1668
1669     def test_POST_FILEURL_check_and_repair(self):
1670         bar_url = self.public_url + "/foo/bar.txt"
1671         d = self.POST(bar_url, t="check", repair="true")
1672         def _check(res):
1673             self.failUnless("Healthy :" in res)
1674         d.addCallback(_check)
1675         redir_url = "http://allmydata.org/TARGET"
1676         def _check2(statuscode, target):
1677             self.failUnlessEqual(statuscode, str(http.FOUND))
1678             self.failUnlessEqual(target, redir_url)
1679         d.addCallback(lambda res:
1680                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1681                                            _check2,
1682                                            self.POST, bar_url,
1683                                            t="check", repair="true",
1684                                            when_done=redir_url))
1685         d.addCallback(lambda res:
1686                       self.POST(bar_url, t="check", return_to=redir_url))
1687         def _check3(res):
1688             self.failUnless("Healthy :" in res)
1689             self.failUnless("Return to parent directory" in res)
1690             self.failUnless(redir_url in res)
1691         d.addCallback(_check3)
1692         return d
1693
1694     def test_POST_DIRURL_check(self):
1695         foo_url = self.public_url + "/foo/"
1696         d = self.POST(foo_url, t="check")
1697         def _check(res):
1698             self.failUnless("Healthy :" in res, res)
1699         d.addCallback(_check)
1700         redir_url = "http://allmydata.org/TARGET"
1701         def _check2(statuscode, target):
1702             self.failUnlessEqual(statuscode, str(http.FOUND))
1703             self.failUnlessEqual(target, redir_url)
1704         d.addCallback(lambda res:
1705                       self.shouldRedirect2("test_POST_DIRURL_check",
1706                                            _check2,
1707                                            self.POST, foo_url,
1708                                            t="check",
1709                                            when_done=redir_url))
1710         d.addCallback(lambda res:
1711                       self.POST(foo_url, t="check", return_to=redir_url))
1712         def _check3(res):
1713             self.failUnless("Healthy :" in res, res)
1714             self.failUnless("Return to parent directory" in res)
1715             self.failUnless(redir_url in res)
1716         d.addCallback(_check3)
1717
1718         d.addCallback(lambda res:
1719                       self.POST(foo_url, t="check", output="JSON"))
1720         def _check_json(res):
1721             data = simplejson.loads(res)
1722             self.failUnless("storage-index" in data)
1723             self.failUnless(data["results"]["healthy"])
1724         d.addCallback(_check_json)
1725
1726         return d
1727
1728     def test_POST_DIRURL_check_and_repair(self):
1729         foo_url = self.public_url + "/foo/"
1730         d = self.POST(foo_url, t="check", repair="true")
1731         def _check(res):
1732             self.failUnless("Healthy :" in res, res)
1733         d.addCallback(_check)
1734         redir_url = "http://allmydata.org/TARGET"
1735         def _check2(statuscode, target):
1736             self.failUnlessEqual(statuscode, str(http.FOUND))
1737             self.failUnlessEqual(target, redir_url)
1738         d.addCallback(lambda res:
1739                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1740                                            _check2,
1741                                            self.POST, foo_url,
1742                                            t="check", repair="true",
1743                                            when_done=redir_url))
1744         d.addCallback(lambda res:
1745                       self.POST(foo_url, t="check", return_to=redir_url))
1746         def _check3(res):
1747             self.failUnless("Healthy :" in res)
1748             self.failUnless("Return to parent directory" in res)
1749             self.failUnless(redir_url in res)
1750         d.addCallback(_check3)
1751         return d
1752
1753     def wait_for_operation(self, ignored, ophandle):
1754         url = "/operations/" + ophandle
1755         url += "?t=status&output=JSON"
1756         d = self.GET(url)
1757         def _got(res):
1758             data = simplejson.loads(res)
1759             if not data["finished"]:
1760                 d = self.stall(delay=1.0)
1761                 d.addCallback(self.wait_for_operation, ophandle)
1762                 return d
1763             return data
1764         d.addCallback(_got)
1765         return d
1766
1767     def get_operation_results(self, ignored, ophandle, output=None):
1768         url = "/operations/" + ophandle
1769         url += "?t=status"
1770         if output:
1771             url += "&output=" + output
1772         d = self.GET(url)
1773         def _got(res):
1774             if output and output.lower() == "json":
1775                 return simplejson.loads(res)
1776             return res
1777         d.addCallback(_got)
1778         return d
1779
1780     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1781         d = self.shouldFail2(error.Error,
1782                              "test_POST_DIRURL_deepcheck_no_ophandle",
1783                              "400 Bad Request",
1784                              "slow operation requires ophandle=",
1785                              self.POST, self.public_url, t="start-deep-check")
1786         return d
1787
1788     def test_POST_DIRURL_deepcheck(self):
1789         def _check_redirect(statuscode, target):
1790             self.failUnlessEqual(statuscode, str(http.FOUND))
1791             self.failUnless(target.endswith("/operations/123"))
1792         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1793                                  self.POST, self.public_url,
1794                                  t="start-deep-check", ophandle="123")
1795         d.addCallback(self.wait_for_operation, "123")
1796         def _check_json(data):
1797             self.failUnlessEqual(data["finished"], True)
1798             self.failUnlessEqual(data["count-objects-checked"], 8)
1799             self.failUnlessEqual(data["count-objects-healthy"], 8)
1800         d.addCallback(_check_json)
1801         d.addCallback(self.get_operation_results, "123", "html")
1802         def _check_html(res):
1803             self.failUnless("Objects Checked: <span>8</span>" in res)
1804             self.failUnless("Objects Healthy: <span>8</span>" in res)
1805         d.addCallback(_check_html)
1806
1807         d.addCallback(lambda res:
1808                       self.GET("/operations/123/"))
1809         d.addCallback(_check_html) # should be the same as without the slash
1810
1811         d.addCallback(lambda res:
1812                       self.shouldFail2(error.Error, "one", "404 Not Found",
1813                                        "No detailed results for SI bogus",
1814                                        self.GET, "/operations/123/bogus"))
1815
1816         foo_si = self._foo_node.get_storage_index()
1817         foo_si_s = base32.b2a(foo_si)
1818         d.addCallback(lambda res:
1819                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1820         def _check_foo_json(res):
1821             data = simplejson.loads(res)
1822             self.failUnlessEqual(data["storage-index"], foo_si_s)
1823             self.failUnless(data["results"]["healthy"])
1824         d.addCallback(_check_foo_json)
1825         return d
1826
1827     def test_POST_DIRURL_deepcheck_and_repair(self):
1828         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1829                       ophandle="124", output="json", followRedirect=True)
1830         d.addCallback(self.wait_for_operation, "124")
1831         def _check_json(data):
1832             self.failUnlessEqual(data["finished"], True)
1833             self.failUnlessEqual(data["count-objects-checked"], 8)
1834             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1835             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1836             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1837             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1838             self.failUnlessEqual(data["count-repairs-successful"], 0)
1839             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1840             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1841             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1842             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1843         d.addCallback(_check_json)
1844         d.addCallback(self.get_operation_results, "124", "html")
1845         def _check_html(res):
1846             self.failUnless("Objects Checked: <span>8</span>" in res)
1847
1848             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1849             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1850             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1851
1852             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1853             self.failUnless("Repairs Successful: <span>0</span>" in res)
1854             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1855
1856             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1857             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1858             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1859         d.addCallback(_check_html)
1860         return d
1861
1862     def test_POST_FILEURL_bad_t(self):
1863         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1864                              "POST to file: bad t=bogus",
1865                              self.POST, self.public_url + "/foo/bar.txt",
1866                              t="bogus")
1867         return d
1868
1869     def test_POST_mkdir(self): # return value?
1870         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1871         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1872         d.addCallback(self.failUnlessNodeKeysAre, [])
1873         return d
1874
1875     def test_POST_mkdir_2(self):
1876         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1877         d.addCallback(lambda res:
1878                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1879         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1880         d.addCallback(self.failUnlessNodeKeysAre, [])
1881         return d
1882
1883     def test_POST_mkdirs_2(self):
1884         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1885         d.addCallback(lambda res:
1886                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1887         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1888         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1889         d.addCallback(self.failUnlessNodeKeysAre, [])
1890         return d
1891
1892     def test_POST_mkdir_no_parentdir_noredirect(self):
1893         d = self.POST("/uri?t=mkdir")
1894         def _after_mkdir(res):
1895             uri.NewDirectoryURI.init_from_string(res)
1896         d.addCallback(_after_mkdir)
1897         return d
1898
1899     def test_POST_mkdir_no_parentdir_redirect(self):
1900         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1901         d.addBoth(self.shouldRedirect, None, statuscode='303')
1902         def _check_target(target):
1903             target = urllib.unquote(target)
1904             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1905         d.addCallback(_check_target)
1906         return d
1907
1908     def test_POST_noparent_bad(self):
1909         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1910                                   "/uri accepts only PUT, PUT?t=mkdir, "
1911                                   "POST?t=upload, and POST?t=mkdir",
1912                                   self.POST, "/uri?t=bogus")
1913         return d
1914
1915     def test_welcome_page_mkdir_button(self):
1916         # Fetch the welcome page.
1917         d = self.GET("/")
1918         def _after_get_welcome_page(res):
1919             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1920             mo = MKDIR_BUTTON_RE.search(res)
1921             formaction = mo.group(1)
1922             formt = mo.group(2)
1923             formaname = mo.group(3)
1924             formavalue = mo.group(4)
1925             return (formaction, formt, formaname, formavalue)
1926         d.addCallback(_after_get_welcome_page)
1927         def _after_parse_form(res):
1928             (formaction, formt, formaname, formavalue) = res
1929             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1930         d.addCallback(_after_parse_form)
1931         d.addBoth(self.shouldRedirect, None, statuscode='303')
1932         return d
1933
1934     def test_POST_mkdir_replace(self): # return value?
1935         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1936         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1937         d.addCallback(self.failUnlessNodeKeysAre, [])
1938         return d
1939
1940     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1941         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1942         d.addBoth(self.shouldFail, error.Error,
1943                   "POST_mkdir_no_replace_queryarg",
1944                   "409 Conflict",
1945                   "There was already a child by that name, and you asked me "
1946                   "to not replace it")
1947         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1948         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1949         return d
1950
1951     def test_POST_mkdir_no_replace_field(self): # return value?
1952         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1953                       replace="false")
1954         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1955                   "409 Conflict",
1956                   "There was already a child by that name, and you asked me "
1957                   "to not replace it")
1958         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1959         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1960         return d
1961
1962     def test_POST_mkdir_whendone_field(self):
1963         d = self.POST(self.public_url + "/foo",
1964                       t="mkdir", name="newdir", when_done="/THERE")
1965         d.addBoth(self.shouldRedirect, "/THERE")
1966         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1967         d.addCallback(self.failUnlessNodeKeysAre, [])
1968         return d
1969
1970     def test_POST_mkdir_whendone_queryarg(self):
1971         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1972                       t="mkdir", name="newdir")
1973         d.addBoth(self.shouldRedirect, "/THERE")
1974         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1975         d.addCallback(self.failUnlessNodeKeysAre, [])
1976         return d
1977
1978     def test_POST_bad_t(self):
1979         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1980                              "POST to a directory with bad t=BOGUS",
1981                              self.POST, self.public_url + "/foo", t="BOGUS")
1982         return d
1983
1984     def test_POST_set_children(self):
1985         contents9, n9, newuri9 = self.makefile(9)
1986         contents10, n10, newuri10 = self.makefile(10)
1987         contents11, n11, newuri11 = self.makefile(11)
1988
1989         reqbody = """{
1990                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1991                                                 "size": 0,
1992                                                 "metadata": {
1993                                                   "ctime": 1002777696.7564139,
1994                                                   "mtime": 1002777696.7564139
1995                                                  }
1996                                                } ],
1997                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1998                                                 "size": 1,
1999                                                 "metadata": {
2000                                                   "ctime": 1002777696.7564139,
2001                                                   "mtime": 1002777696.7564139
2002                                                  }
2003                                                } ],
2004                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2005                                                 "size": 2,
2006                                                 "metadata": {
2007                                                   "ctime": 1002777696.7564139,
2008                                                   "mtime": 1002777696.7564139
2009                                                  }
2010                                                } ]
2011                     }""" % (newuri9, newuri10, newuri11)
2012
2013         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2014
2015         d = client.getPage(url, method="POST", postdata=reqbody)
2016         def _then(res):
2017             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2018             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2019             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2020
2021         d.addCallback(_then)
2022         d.addErrback(self.dump_error)
2023         return d
2024
2025     def test_POST_put_uri(self):
2026         contents, n, newuri = self.makefile(8)
2027         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2028         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2029         d.addCallback(lambda res:
2030                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2031                                                       contents))
2032         return d
2033
2034     def test_POST_put_uri_replace(self):
2035         contents, n, newuri = self.makefile(8)
2036         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2037         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2038         d.addCallback(lambda res:
2039                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2040                                                       contents))
2041         return d
2042
2043     def test_POST_put_uri_no_replace_queryarg(self):
2044         contents, n, newuri = self.makefile(8)
2045         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2046                       name="bar.txt", uri=newuri)
2047         d.addBoth(self.shouldFail, error.Error,
2048                   "POST_put_uri_no_replace_queryarg",
2049                   "409 Conflict",
2050                   "There was already a child by that name, and you asked me "
2051                   "to not replace it")
2052         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2053         d.addCallback(self.failUnlessIsBarDotTxt)
2054         return d
2055
2056     def test_POST_put_uri_no_replace_field(self):
2057         contents, n, newuri = self.makefile(8)
2058         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2059                       name="bar.txt", uri=newuri)
2060         d.addBoth(self.shouldFail, error.Error,
2061                   "POST_put_uri_no_replace_field",
2062                   "409 Conflict",
2063                   "There was already a child by that name, and you asked me "
2064                   "to not replace it")
2065         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2066         d.addCallback(self.failUnlessIsBarDotTxt)
2067         return d
2068
2069     def test_POST_delete(self):
2070         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2071         d.addCallback(lambda res: self._foo_node.list())
2072         def _check(children):
2073             self.failIf(u"bar.txt" in children)
2074         d.addCallback(_check)
2075         return d
2076
2077     def test_POST_rename_file(self):
2078         d = self.POST(self.public_url + "/foo", t="rename",
2079                       from_name="bar.txt", to_name='wibble.txt')
2080         d.addCallback(lambda res:
2081                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2082         d.addCallback(lambda res:
2083                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2084         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2085         d.addCallback(self.failUnlessIsBarDotTxt)
2086         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2087         d.addCallback(self.failUnlessIsBarJSON)
2088         return d
2089
2090     def test_POST_rename_file_redundant(self):
2091         d = self.POST(self.public_url + "/foo", t="rename",
2092                       from_name="bar.txt", to_name='bar.txt')
2093         d.addCallback(lambda res:
2094                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2095         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2096         d.addCallback(self.failUnlessIsBarDotTxt)
2097         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2098         d.addCallback(self.failUnlessIsBarJSON)
2099         return d
2100
2101     def test_POST_rename_file_replace(self):
2102         # rename a file and replace a directory with it
2103         d = self.POST(self.public_url + "/foo", t="rename",
2104                       from_name="bar.txt", to_name='empty')
2105         d.addCallback(lambda res:
2106                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2107         d.addCallback(lambda res:
2108                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2109         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2110         d.addCallback(self.failUnlessIsBarDotTxt)
2111         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2112         d.addCallback(self.failUnlessIsBarJSON)
2113         return d
2114
2115     def test_POST_rename_file_no_replace_queryarg(self):
2116         # rename a file and replace a directory with it
2117         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2118                       from_name="bar.txt", to_name='empty')
2119         d.addBoth(self.shouldFail, error.Error,
2120                   "POST_rename_file_no_replace_queryarg",
2121                   "409 Conflict",
2122                   "There was already a child by that name, and you asked me "
2123                   "to not replace it")
2124         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2125         d.addCallback(self.failUnlessIsEmptyJSON)
2126         return d
2127
2128     def test_POST_rename_file_no_replace_field(self):
2129         # rename a file and replace a directory with it
2130         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2131                       from_name="bar.txt", to_name='empty')
2132         d.addBoth(self.shouldFail, error.Error,
2133                   "POST_rename_file_no_replace_field",
2134                   "409 Conflict",
2135                   "There was already a child by that name, and you asked me "
2136                   "to not replace it")
2137         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2138         d.addCallback(self.failUnlessIsEmptyJSON)
2139         return d
2140
2141     def failUnlessIsEmptyJSON(self, res):
2142         data = simplejson.loads(res)
2143         self.failUnlessEqual(data[0], "dirnode", data)
2144         self.failUnlessEqual(len(data[1]["children"]), 0)
2145
2146     def test_POST_rename_file_slash_fail(self):
2147         d = self.POST(self.public_url + "/foo", t="rename",
2148                       from_name="bar.txt", to_name='kirk/spock.txt')
2149         d.addBoth(self.shouldFail, error.Error,
2150                   "test_POST_rename_file_slash_fail",
2151                   "400 Bad Request",
2152                   "to_name= may not contain a slash",
2153                   )
2154         d.addCallback(lambda res:
2155                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2156         return d
2157
2158     def test_POST_rename_dir(self):
2159         d = self.POST(self.public_url, t="rename",
2160                       from_name="foo", to_name='plunk')
2161         d.addCallback(lambda res:
2162                       self.failIfNodeHasChild(self.public_root, u"foo"))
2163         d.addCallback(lambda res:
2164                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2165         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2166         d.addCallback(self.failUnlessIsFooJSON)
2167         return d
2168
2169     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2170         """ If target is not None then the redirection has to go to target.  If
2171         statuscode is not None then the redirection has to be accomplished with
2172         that HTTP status code."""
2173         if not isinstance(res, failure.Failure):
2174             to_where = (target is None) and "somewhere" or ("to " + target)
2175             self.fail("%s: we were expecting to get redirected %s, not get an"
2176                       " actual page: %s" % (which, to_where, res))
2177         res.trap(error.PageRedirect)
2178         if statuscode is not None:
2179             self.failUnlessEqual(res.value.status, statuscode,
2180                                  "%s: not a redirect" % which)
2181         if target is not None:
2182             # the PageRedirect does not seem to capture the uri= query arg
2183             # properly, so we can't check for it.
2184             realtarget = self.webish_url + target
2185             self.failUnlessEqual(res.value.location, realtarget,
2186                                  "%s: wrong target" % which)
2187         return res.value.location
2188
2189     def test_GET_URI_form(self):
2190         base = "/uri?uri=%s" % self._bar_txt_uri
2191         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2192         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2193         d = self.GET(base)
2194         d.addBoth(self.shouldRedirect, targetbase)
2195         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2196         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2197         d.addCallback(lambda res: self.GET(base+"&t=json"))
2198         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2199         d.addCallback(self.log, "about to get file by uri")
2200         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2201         d.addCallback(self.failUnlessIsBarDotTxt)
2202         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2203         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2204                                            followRedirect=True))
2205         d.addCallback(self.failUnlessIsFooJSON)
2206         d.addCallback(self.log, "got dir by uri")
2207
2208         return d
2209
2210     def test_GET_URI_form_bad(self):
2211         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2212                              "400 Bad Request", "GET /uri requires uri=",
2213                              self.GET, "/uri")
2214         return d
2215
2216     def test_GET_rename_form(self):
2217         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2218                      followRedirect=True)
2219         def _check(res):
2220             self.failUnless('name="when_done" value="."' in res, res)
2221             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2222         d.addCallback(_check)
2223         return d
2224
2225     def log(self, res, msg):
2226         #print "MSG: %s  RES: %s" % (msg, res)
2227         log.msg(msg)
2228         return res
2229
2230     def test_GET_URI_URL(self):
2231         base = "/uri/%s" % self._bar_txt_uri
2232         d = self.GET(base)
2233         d.addCallback(self.failUnlessIsBarDotTxt)
2234         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2235         d.addCallback(self.failUnlessIsBarDotTxt)
2236         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2237         d.addCallback(self.failUnlessIsBarDotTxt)
2238         return d
2239
2240     def test_GET_URI_URL_dir(self):
2241         base = "/uri/%s?t=json" % self._foo_uri
2242         d = self.GET(base)
2243         d.addCallback(self.failUnlessIsFooJSON)
2244         return d
2245
2246     def test_GET_URI_URL_missing(self):
2247         base = "/uri/%s" % self._bad_file_uri
2248         d = self.GET(base)
2249         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2250                   http.GONE, response_substring="NotEnoughSharesError")
2251         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2252         # here? we must arrange for a download to fail after target.open()
2253         # has been called, and then inspect the response to see that it is
2254         # shorter than we expected.
2255         return d
2256
2257     def test_PUT_DIRURL_uri(self):
2258         d = self.s.create_empty_dirnode()
2259         def _made_dir(dn):
2260             new_uri = dn.get_uri()
2261             # replace /foo with a new (empty) directory
2262             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2263             d.addCallback(lambda res:
2264                           self.failUnlessEqual(res.strip(), new_uri))
2265             d.addCallback(lambda res:
2266                           self.failUnlessChildURIIs(self.public_root,
2267                                                     u"foo",
2268                                                     new_uri))
2269             return d
2270         d.addCallback(_made_dir)
2271         return d
2272
2273     def test_PUT_DIRURL_uri_noreplace(self):
2274         d = self.s.create_empty_dirnode()
2275         def _made_dir(dn):
2276             new_uri = dn.get_uri()
2277             # replace /foo with a new (empty) directory, but ask that
2278             # replace=false, so it should fail
2279             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2280                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2281                                  self.PUT,
2282                                  self.public_url + "/foo?t=uri&replace=false",
2283                                  new_uri)
2284             d.addCallback(lambda res:
2285                           self.failUnlessChildURIIs(self.public_root,
2286                                                     u"foo",
2287                                                     self._foo_uri))
2288             return d
2289         d.addCallback(_made_dir)
2290         return d
2291
2292     def test_PUT_DIRURL_bad_t(self):
2293         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2294                                  "400 Bad Request", "PUT to a directory",
2295                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2296         d.addCallback(lambda res:
2297                       self.failUnlessChildURIIs(self.public_root,
2298                                                 u"foo",
2299                                                 self._foo_uri))
2300         return d
2301
2302     def test_PUT_NEWFILEURL_uri(self):
2303         contents, n, new_uri = self.makefile(8)
2304         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2305         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2306         d.addCallback(lambda res:
2307                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2308                                                       contents))
2309         return d
2310
2311     def test_PUT_NEWFILEURL_uri_replace(self):
2312         contents, n, new_uri = self.makefile(8)
2313         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2314         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2315         d.addCallback(lambda res:
2316                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2317                                                       contents))
2318         return d
2319
2320     def test_PUT_NEWFILEURL_uri_no_replace(self):
2321         contents, n, new_uri = self.makefile(8)
2322         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2323         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2324                   "409 Conflict",
2325                   "There was already a child by that name, and you asked me "
2326                   "to not replace it")
2327         return d
2328
2329     def test_PUT_NEWFILE_URI(self):
2330         file_contents = "New file contents here\n"
2331         d = self.PUT("/uri", file_contents)
2332         def _check(uri):
2333             assert isinstance(uri, str), uri
2334             self.failUnless(uri in FakeCHKFileNode.all_contents)
2335             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2336                                  file_contents)
2337             return self.GET("/uri/%s" % uri)
2338         d.addCallback(_check)
2339         def _check2(res):
2340             self.failUnlessEqual(res, file_contents)
2341         d.addCallback(_check2)
2342         return d
2343
2344     def test_PUT_NEWFILE_URI_only_PUT(self):
2345         d = self.PUT("/uri?t=bogus", "")
2346         d.addBoth(self.shouldFail, error.Error,
2347                   "PUT_NEWFILE_URI_only_PUT",
2348                   "400 Bad Request",
2349                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2350         return d
2351
2352     def test_PUT_NEWFILE_URI_mutable(self):
2353         file_contents = "New file contents here\n"
2354         d = self.PUT("/uri?mutable=true", file_contents)
2355         def _check_mutable(uri):
2356             uri = uri.strip()
2357             u = IURI(uri)
2358             self.failUnless(IMutableFileURI.providedBy(u))
2359             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2360             n = self.s.create_node_from_uri(uri)
2361             return n.download_best_version()
2362         d.addCallback(_check_mutable)
2363         def _check2_mutable(data):
2364             self.failUnlessEqual(data, file_contents)
2365         d.addCallback(_check2_mutable)
2366         return d
2367
2368         def _check(uri):
2369             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2370             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2371                                  file_contents)
2372             return self.GET("/uri/%s" % uri)
2373         d.addCallback(_check)
2374         def _check2(res):
2375             self.failUnlessEqual(res, file_contents)
2376         d.addCallback(_check2)
2377         return d
2378
2379     def test_PUT_mkdir(self):
2380         d = self.PUT("/uri?t=mkdir", "")
2381         def _check(uri):
2382             n = self.s.create_node_from_uri(uri.strip())
2383             d2 = self.failUnlessNodeKeysAre(n, [])
2384             d2.addCallback(lambda res:
2385                            self.GET("/uri/%s?t=json" % uri))
2386             return d2
2387         d.addCallback(_check)
2388         d.addCallback(self.failUnlessIsEmptyJSON)
2389         return d
2390
2391     def test_POST_check(self):
2392         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2393         def _done(res):
2394             # this returns a string form of the results, which are probably
2395             # None since we're using fake filenodes.
2396             # TODO: verify that the check actually happened, by changing
2397             # FakeCHKFileNode to count how many times .check() has been
2398             # called.
2399             pass
2400         d.addCallback(_done)
2401         return d
2402
2403     def test_bad_method(self):
2404         url = self.webish_url + self.public_url + "/foo/bar.txt"
2405         d = self.shouldHTTPError2("test_bad_method",
2406                                   501, "Not Implemented",
2407                                   "I don't know how to treat a BOGUS request.",
2408                                   client.getPage, url, method="BOGUS")
2409         return d
2410
2411     def test_short_url(self):
2412         url = self.webish_url + "/uri"
2413         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2414                                   "I don't know how to treat a DELETE request.",
2415                                   client.getPage, url, method="DELETE")
2416         return d
2417
2418     def test_ophandle_bad(self):
2419         url = self.webish_url + "/operations/bogus?t=status"
2420         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2421                                   "unknown/expired handle 'bogus'",
2422                                   client.getPage, url)
2423         return d
2424
2425     def test_ophandle_cancel(self):
2426         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2427                       followRedirect=True)
2428         d.addCallback(lambda ignored:
2429                       self.GET("/operations/128?t=status&output=JSON"))
2430         def _check1(res):
2431             data = simplejson.loads(res)
2432             self.failUnless("finished" in data, res)
2433             monitor = self.ws.root.child_operations.handles["128"][0]
2434             d = self.POST("/operations/128?t=cancel&output=JSON")
2435             def _check2(res):
2436                 data = simplejson.loads(res)
2437                 self.failUnless("finished" in data, res)
2438                 # t=cancel causes the handle to be forgotten
2439                 self.failUnless(monitor.is_cancelled())
2440             d.addCallback(_check2)
2441             return d
2442         d.addCallback(_check1)
2443         d.addCallback(lambda ignored:
2444                       self.shouldHTTPError2("test_ophandle_cancel",
2445                                             404, "404 Not Found",
2446                                             "unknown/expired handle '128'",
2447                                             self.GET,
2448                                             "/operations/128?t=status&output=JSON"))
2449         return d
2450
2451     def test_ophandle_retainfor(self):
2452         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2453                       followRedirect=True)
2454         d.addCallback(lambda ignored:
2455                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2456         def _check1(res):
2457             data = simplejson.loads(res)
2458             self.failUnless("finished" in data, res)
2459         d.addCallback(_check1)
2460         # the retain-for=0 will cause the handle to be expired very soon
2461         d.addCallback(self.stall, 2.0)
2462         d.addCallback(lambda ignored:
2463                       self.shouldHTTPError2("test_ophandle_retainfor",
2464                                             404, "404 Not Found",
2465                                             "unknown/expired handle '129'",
2466                                             self.GET,
2467                                             "/operations/129?t=status&output=JSON"))
2468         return d
2469
2470     def test_ophandle_release_after_complete(self):
2471         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2472                       followRedirect=True)
2473         d.addCallback(self.wait_for_operation, "130")
2474         d.addCallback(lambda ignored:
2475                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2476         # the release-after-complete=true will cause the handle to be expired
2477         d.addCallback(lambda ignored:
2478                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2479                                             404, "404 Not Found",
2480                                             "unknown/expired handle '130'",
2481                                             self.GET,
2482                                             "/operations/130?t=status&output=JSON"))
2483         return d
2484
2485     def test_incident(self):
2486         d = self.POST("/report_incident", details="eek")
2487         def _done(res):
2488             self.failUnless("Thank you for your report!" in res, res)
2489         d.addCallback(_done)
2490         return d
2491
2492     def test_static(self):
2493         webdir = os.path.join(self.staticdir, "subdir")
2494         fileutil.make_dirs(webdir)
2495         f = open(os.path.join(webdir, "hello.txt"), "wb")
2496         f.write("hello")
2497         f.close()
2498
2499         d = self.GET("/static/subdir/hello.txt")
2500         def _check(res):
2501             self.failUnlessEqual(res, "hello")
2502         d.addCallback(_check)
2503         return d
2504
2505
2506 class Util(unittest.TestCase):
2507     def test_abbreviate_time(self):
2508         self.failUnlessEqual(common.abbreviate_time(None), "")
2509         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2510         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2511         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2512         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2513
2514     def test_abbreviate_rate(self):
2515         self.failUnlessEqual(common.abbreviate_rate(None), "")
2516         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2517         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2518         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2519
2520     def test_abbreviate_size(self):
2521         self.failUnlessEqual(common.abbreviate_size(None), "")
2522         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2523         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2524         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2525         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2526
2527
2528 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase):
2529
2530     def GET(self, urlpath, followRedirect=False, return_response=False,
2531             method="GET", clientnum=0, **kwargs):
2532         # if return_response=True, this fires with (data, statuscode,
2533         # respheaders) instead of just data.
2534         assert not isinstance(urlpath, unicode)
2535         url = self.client_baseurls[clientnum] + urlpath
2536         factory = HTTPClientGETFactory(url, method=method,
2537                                        followRedirect=followRedirect, **kwargs)
2538         reactor.connectTCP("localhost", self.client_webports[clientnum],factory)
2539         d = factory.deferred
2540         def _got_data(data):
2541             return (data, factory.status, factory.response_headers)
2542         if return_response:
2543             d.addCallback(_got_data)
2544         return factory.deferred
2545
2546     def CHECK(self, ign, which, args, clientnum=0):
2547         fileurl = self.fileurls[which]
2548         url = fileurl + "?" + args
2549         return self.GET(url, method="POST", clientnum=clientnum)
2550
2551     def test_filecheck(self):
2552         self.basedir = "web/Grid/filecheck"
2553         self.set_up_grid()
2554         c0 = self.g.clients[0]
2555         self.uris = {}
2556         DATA = "data" * 100
2557         d = c0.upload(upload.Data(DATA, convergence=""))
2558         def _stash_uri(ur, which):
2559             self.uris[which] = ur.uri
2560         d.addCallback(_stash_uri, "good")
2561         d.addCallback(lambda ign:
2562                       c0.upload(upload.Data(DATA+"1", convergence="")))
2563         d.addCallback(_stash_uri, "sick")
2564         d.addCallback(lambda ign:
2565                       c0.upload(upload.Data(DATA+"2", convergence="")))
2566         d.addCallback(_stash_uri, "dead")
2567         def _stash_mutable_uri(n, which):
2568             self.uris[which] = n.get_uri()
2569             assert isinstance(self.uris[which], str)
2570         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2571         d.addCallback(_stash_mutable_uri, "corrupt")
2572         d.addCallback(lambda ign:
2573                       c0.upload(upload.Data("literal", convergence="")))
2574         d.addCallback(_stash_uri, "small")
2575
2576         def _compute_fileurls(ignored):
2577             self.fileurls = {}
2578             for which in self.uris:
2579                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2580         d.addCallback(_compute_fileurls)
2581
2582         def _clobber_shares(ignored):
2583             good_shares = self.find_shares(self.uris["good"])
2584             self.failUnlessEqual(len(good_shares), 10)
2585             sick_shares = self.find_shares(self.uris["sick"])
2586             os.unlink(sick_shares[0][2])
2587             dead_shares = self.find_shares(self.uris["dead"])
2588             for i in range(1, 10):
2589                 os.unlink(dead_shares[i][2])
2590             c_shares = self.find_shares(self.uris["corrupt"])
2591             cso = CorruptShareOptions()
2592             cso.stdout = StringIO()
2593             cso.parseOptions([c_shares[0][2]])
2594             corrupt_share(cso)
2595         d.addCallback(_clobber_shares)
2596
2597         d.addCallback(self.CHECK, "good", "t=check")
2598         def _got_html_good(res):
2599             self.failUnless("Healthy" in res, res)
2600             self.failIf("Not Healthy" in res, res)
2601         d.addCallback(_got_html_good)
2602         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2603         def _got_html_good_return_to(res):
2604             self.failUnless("Healthy" in res, res)
2605             self.failIf("Not Healthy" in res, res)
2606             self.failUnless('<a href="somewhere">Return to parent directory'
2607                             in res, res)
2608         d.addCallback(_got_html_good_return_to)
2609         d.addCallback(self.CHECK, "good", "t=check&output=json")
2610         def _got_json_good(res):
2611             r = simplejson.loads(res)
2612             self.failUnlessEqual(r["summary"], "Healthy")
2613             self.failUnless(r["results"]["healthy"])
2614             self.failIf(r["results"]["needs-rebalancing"])
2615             self.failUnless(r["results"]["recoverable"])
2616         d.addCallback(_got_json_good)
2617
2618         d.addCallback(self.CHECK, "small", "t=check")
2619         def _got_html_small(res):
2620             self.failUnless("Literal files are always healthy" in res, res)
2621             self.failIf("Not Healthy" in res, res)
2622         d.addCallback(_got_html_small)
2623         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2624         def _got_html_small_return_to(res):
2625             self.failUnless("Literal files are always healthy" in res, res)
2626             self.failIf("Not Healthy" in res, res)
2627             self.failUnless('<a href="somewhere">Return to parent directory'
2628                             in res, res)
2629         d.addCallback(_got_html_small_return_to)
2630         d.addCallback(self.CHECK, "small", "t=check&output=json")
2631         def _got_json_small(res):
2632             r = simplejson.loads(res)
2633             self.failUnlessEqual(r["storage-index"], "")
2634             self.failUnless(r["results"]["healthy"])
2635         d.addCallback(_got_json_small)
2636
2637         d.addCallback(self.CHECK, "sick", "t=check")
2638         def _got_html_sick(res):
2639             self.failUnless("Not Healthy" in res, res)
2640         d.addCallback(_got_html_sick)
2641         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2642         def _got_json_sick(res):
2643             r = simplejson.loads(res)
2644             self.failUnlessEqual(r["summary"],
2645                                  "Not Healthy: 9 shares (enc 3-of-10)")
2646             self.failIf(r["results"]["healthy"])
2647             self.failIf(r["results"]["needs-rebalancing"])
2648             self.failUnless(r["results"]["recoverable"])
2649         d.addCallback(_got_json_sick)
2650
2651         d.addCallback(self.CHECK, "dead", "t=check")
2652         def _got_html_dead(res):
2653             self.failUnless("Not Healthy" in res, res)
2654         d.addCallback(_got_html_dead)
2655         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2656         def _got_json_dead(res):
2657             r = simplejson.loads(res)
2658             self.failUnlessEqual(r["summary"],
2659                                  "Not Healthy: 1 shares (enc 3-of-10)")
2660             self.failIf(r["results"]["healthy"])
2661             self.failIf(r["results"]["needs-rebalancing"])
2662             self.failIf(r["results"]["recoverable"])
2663         d.addCallback(_got_json_dead)
2664
2665         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2666         def _got_html_corrupt(res):
2667             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2668         d.addCallback(_got_html_corrupt)
2669         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2670         def _got_json_corrupt(res):
2671             r = simplejson.loads(res)
2672             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2673                             r["summary"])
2674             self.failIf(r["results"]["healthy"])
2675             self.failUnless(r["results"]["recoverable"])
2676             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2677             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2678         d.addCallback(_got_json_corrupt)
2679
2680         d.addErrback(self.explain_web_error)
2681         return d
2682
2683     def test_repair_html(self):
2684         self.basedir = "web/Grid/repair_html"
2685         self.set_up_grid()
2686         c0 = self.g.clients[0]
2687         self.uris = {}
2688         DATA = "data" * 100
2689         d = c0.upload(upload.Data(DATA, convergence=""))
2690         def _stash_uri(ur, which):
2691             self.uris[which] = ur.uri
2692         d.addCallback(_stash_uri, "good")
2693         d.addCallback(lambda ign:
2694                       c0.upload(upload.Data(DATA+"1", convergence="")))
2695         d.addCallback(_stash_uri, "sick")
2696         d.addCallback(lambda ign:
2697                       c0.upload(upload.Data(DATA+"2", convergence="")))
2698         d.addCallback(_stash_uri, "dead")
2699         def _stash_mutable_uri(n, which):
2700             self.uris[which] = n.get_uri()
2701             assert isinstance(self.uris[which], str)
2702         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2703         d.addCallback(_stash_mutable_uri, "corrupt")
2704
2705         def _compute_fileurls(ignored):
2706             self.fileurls = {}
2707             for which in self.uris:
2708                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2709         d.addCallback(_compute_fileurls)
2710
2711         def _clobber_shares(ignored):
2712             good_shares = self.find_shares(self.uris["good"])
2713             self.failUnlessEqual(len(good_shares), 10)
2714             sick_shares = self.find_shares(self.uris["sick"])
2715             os.unlink(sick_shares[0][2])
2716             dead_shares = self.find_shares(self.uris["dead"])
2717             for i in range(1, 10):
2718                 os.unlink(dead_shares[i][2])
2719             c_shares = self.find_shares(self.uris["corrupt"])
2720             cso = CorruptShareOptions()
2721             cso.stdout = StringIO()
2722             cso.parseOptions([c_shares[0][2]])
2723             corrupt_share(cso)
2724         d.addCallback(_clobber_shares)
2725
2726         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2727         def _got_html_good(res):
2728             self.failUnless("Healthy" in res, res)
2729             self.failIf("Not Healthy" in res, res)
2730             self.failUnless("No repair necessary" in res, res)
2731         d.addCallback(_got_html_good)
2732
2733         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2734         def _got_html_sick(res):
2735             self.failUnless("Healthy : healthy" in res, res)
2736             self.failIf("Not Healthy" in res, res)
2737             self.failUnless("Repair successful" in res, res)
2738         d.addCallback(_got_html_sick)
2739
2740         # repair of a dead file will fail, of course, but it isn't yet
2741         # clear how this should be reported. Right now it shows up as
2742         # a "410 Gone".
2743         #
2744         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2745         #def _got_html_dead(res):
2746         #    print res
2747         #    self.failUnless("Healthy : healthy" in res, res)
2748         #    self.failIf("Not Healthy" in res, res)
2749         #    self.failUnless("No repair necessary" in res, res)
2750         #d.addCallback(_got_html_dead)
2751
2752         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2753         def _got_html_corrupt(res):
2754             self.failUnless("Healthy : Healthy" in res, res)
2755             self.failIf("Not Healthy" in res, res)
2756             self.failUnless("Repair successful" in res, res)
2757         d.addCallback(_got_html_corrupt)
2758
2759         d.addErrback(self.explain_web_error)
2760         return d
2761
2762     def test_repair_json(self):
2763         self.basedir = "web/Grid/repair_json"
2764         self.set_up_grid()
2765         c0 = self.g.clients[0]
2766         self.uris = {}
2767         DATA = "data" * 100
2768         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2769         def _stash_uri(ur, which):
2770             self.uris[which] = ur.uri
2771         d.addCallback(_stash_uri, "sick")
2772
2773         def _compute_fileurls(ignored):
2774             self.fileurls = {}
2775             for which in self.uris:
2776                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2777         d.addCallback(_compute_fileurls)
2778
2779         def _clobber_shares(ignored):
2780             sick_shares = self.find_shares(self.uris["sick"])
2781             os.unlink(sick_shares[0][2])
2782         d.addCallback(_clobber_shares)
2783
2784         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2785         def _got_json_sick(res):
2786             r = simplejson.loads(res)
2787             self.failUnlessEqual(r["repair-attempted"], True)
2788             self.failUnlessEqual(r["repair-successful"], True)
2789             self.failUnlessEqual(r["pre-repair-results"]["summary"],
2790                                  "Not Healthy: 9 shares (enc 3-of-10)")
2791             self.failIf(r["pre-repair-results"]["results"]["healthy"])
2792             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2793             self.failUnless(r["post-repair-results"]["results"]["healthy"])
2794         d.addCallback(_got_json_sick)
2795
2796         d.addErrback(self.explain_web_error)
2797         return d
2798
2799     def test_deep_check(self):
2800         self.basedir = "web/Grid/deep_check"
2801         self.set_up_grid()
2802         c0 = self.g.clients[0]
2803         self.uris = {}
2804         self.fileurls = {}
2805         DATA = "data" * 100
2806         d = c0.create_empty_dirnode()
2807         def _stash_root_and_create_file(n):
2808             self.rootnode = n
2809             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2810             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2811         d.addCallback(_stash_root_and_create_file)
2812         def _stash_uri(fn, which):
2813             self.uris[which] = fn.get_uri()
2814         d.addCallback(_stash_uri, "good")
2815         d.addCallback(lambda ign:
2816                       self.rootnode.add_file(u"small",
2817                                              upload.Data("literal",
2818                                                         convergence="")))
2819         d.addCallback(_stash_uri, "small")
2820
2821         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2822         def _done(res):
2823             units = [simplejson.loads(line)
2824                      for line in res.splitlines()
2825                      if line]
2826             self.failUnlessEqual(len(units), 3+1)
2827             # should be parent-first
2828             u0 = units[0]
2829             self.failUnlessEqual(u0["path"], [])
2830             self.failUnlessEqual(u0["type"], "directory")
2831             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2832             u0cr = u0["check-results"]
2833             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2834
2835             ugood = [u for u in units
2836                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2837             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2838             ugoodcr = ugood["check-results"]
2839             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2840
2841             stats = units[-1]
2842             self.failUnlessEqual(stats["type"], "stats")
2843             s = stats["stats"]
2844             self.failUnlessEqual(s["count-immutable-files"], 1)
2845             self.failUnlessEqual(s["count-literal-files"], 1)
2846             self.failUnlessEqual(s["count-directories"], 1)
2847         d.addCallback(_done)
2848
2849         d.addErrback(self.explain_web_error)
2850         return d
2851
2852     def test_deep_check_and_repair(self):
2853         self.basedir = "web/Grid/deep_check_and_repair"
2854         self.set_up_grid()
2855         c0 = self.g.clients[0]
2856         self.uris = {}
2857         self.fileurls = {}
2858         DATA = "data" * 100
2859         d = c0.create_empty_dirnode()
2860         def _stash_root_and_create_file(n):
2861             self.rootnode = n
2862             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2863             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2864         d.addCallback(_stash_root_and_create_file)
2865         def _stash_uri(fn, which):
2866             self.uris[which] = fn.get_uri()
2867         d.addCallback(_stash_uri, "good")
2868         d.addCallback(lambda ign:
2869                       self.rootnode.add_file(u"small",
2870                                              upload.Data("literal",
2871                                                         convergence="")))
2872         d.addCallback(_stash_uri, "small")
2873         d.addCallback(lambda ign:
2874                       self.rootnode.add_file(u"sick",
2875                                              upload.Data(DATA+"1",
2876                                                         convergence="")))
2877         d.addCallback(_stash_uri, "sick")
2878         #d.addCallback(lambda ign:
2879         #              self.rootnode.add_file(u"dead",
2880         #                                     upload.Data(DATA+"2",
2881         #                                                convergence="")))
2882         #d.addCallback(_stash_uri, "dead")
2883
2884         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2885         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2886         #d.addCallback(_stash_uri, "corrupt")
2887
2888         def _clobber_shares(ignored):
2889             good_shares = self.find_shares(self.uris["good"])
2890             self.failUnlessEqual(len(good_shares), 10)
2891             sick_shares = self.find_shares(self.uris["sick"])
2892             os.unlink(sick_shares[0][2])
2893             #dead_shares = self.find_shares(self.uris["dead"])
2894             #for i in range(1, 10):
2895             #    os.unlink(dead_shares[i][2])
2896
2897             #c_shares = self.find_shares(self.uris["corrupt"])
2898             #cso = CorruptShareOptions()
2899             #cso.stdout = StringIO()
2900             #cso.parseOptions([c_shares[0][2]])
2901             #corrupt_share(cso)
2902         d.addCallback(_clobber_shares)
2903
2904         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2905         def _done(res):
2906             units = [simplejson.loads(line)
2907                      for line in res.splitlines()
2908                      if line]
2909             self.failUnlessEqual(len(units), 4+1)
2910             # should be parent-first
2911             u0 = units[0]
2912             self.failUnlessEqual(u0["path"], [])
2913             self.failUnlessEqual(u0["type"], "directory")
2914             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2915             u0crr = u0["check-and-repair-results"]
2916             self.failUnlessEqual(u0crr["repair-attempted"], False)
2917             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2918
2919             ugood = [u for u in units
2920                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2921             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2922             ugoodcrr = ugood["check-and-repair-results"]
2923             self.failUnlessEqual(u0crr["repair-attempted"], False)
2924             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2925
2926             usick = [u for u in units
2927                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
2928             self.failUnlessEqual(usick["cap"], self.uris["sick"])
2929             usickcrr = usick["check-and-repair-results"]
2930             self.failUnlessEqual(usickcrr["repair-attempted"], True)
2931             self.failUnlessEqual(usickcrr["repair-successful"], True)
2932             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2933             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2934
2935             stats = units[-1]
2936             self.failUnlessEqual(stats["type"], "stats")
2937             s = stats["stats"]
2938             self.failUnlessEqual(s["count-immutable-files"], 2)
2939             self.failUnlessEqual(s["count-literal-files"], 1)
2940             self.failUnlessEqual(s["count-directories"], 1)
2941         d.addCallback(_done)
2942
2943         d.addErrback(self.explain_web_error)
2944         return d
2945
2946     def _count_leases(self, ignored, which):
2947         u = self.uris[which]
2948         shares = self.find_shares(u)
2949         lease_counts = []
2950         for shnum, serverid, fn in shares:
2951             if u.startswith("URI:SSK") or u.startswith("URI:DIR2"):
2952                 sf = MutableShareFile(fn)
2953                 num_leases = len(sf.debug_get_leases())
2954             elif u.startswith("URI:CHK"):
2955                 sf = ShareFile(fn)
2956                 num_leases = len(list(sf.iter_leases()))
2957             else:
2958                 raise RuntimeError("can't get leases on %s" % u)
2959         lease_counts.append( (fn, num_leases) )
2960         return lease_counts
2961
2962     def _assert_leasecount(self, lease_counts, expected):
2963         for (fn, num_leases) in lease_counts:
2964             if num_leases != expected:
2965                 self.fail("expected %d leases, have %d, on %s" %
2966                           (expected, num_leases, fn))
2967
2968     def test_add_lease(self):
2969         self.basedir = "web/Grid/add_lease"
2970         self.set_up_grid(num_clients=2)
2971         c0 = self.g.clients[0]
2972         self.uris = {}
2973         DATA = "data" * 100
2974         d = c0.upload(upload.Data(DATA, convergence=""))
2975         def _stash_uri(ur, which):
2976             self.uris[which] = ur.uri
2977         d.addCallback(_stash_uri, "one")
2978         d.addCallback(lambda ign:
2979                       c0.upload(upload.Data(DATA+"1", convergence="")))
2980         d.addCallback(_stash_uri, "two")
2981         def _stash_mutable_uri(n, which):
2982             self.uris[which] = n.get_uri()
2983             assert isinstance(self.uris[which], str)
2984         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
2985         d.addCallback(_stash_mutable_uri, "mutable")
2986
2987         def _compute_fileurls(ignored):
2988             self.fileurls = {}
2989             for which in self.uris:
2990                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2991         d.addCallback(_compute_fileurls)
2992
2993         d.addCallback(self._count_leases, "one")
2994         d.addCallback(self._assert_leasecount, 1)
2995         d.addCallback(self._count_leases, "two")
2996         d.addCallback(self._assert_leasecount, 1)
2997         d.addCallback(self._count_leases, "mutable")
2998         d.addCallback(self._assert_leasecount, 1)
2999
3000         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3001         def _got_html_good(res):
3002             self.failUnless("Healthy" in res, res)
3003             self.failIf("Not Healthy" in res, res)
3004         d.addCallback(_got_html_good)
3005
3006         d.addCallback(self._count_leases, "one")
3007         d.addCallback(self._assert_leasecount, 1)
3008         d.addCallback(self._count_leases, "two")
3009         d.addCallback(self._assert_leasecount, 1)
3010         d.addCallback(self._count_leases, "mutable")
3011         d.addCallback(self._assert_leasecount, 1)
3012
3013         # this CHECK uses the original client, which uses the same
3014         # lease-secrets, so it will just renew the original lease
3015         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3016         d.addCallback(_got_html_good)
3017
3018         d.addCallback(self._count_leases, "one")
3019         d.addCallback(self._assert_leasecount, 1)
3020         d.addCallback(self._count_leases, "two")
3021         d.addCallback(self._assert_leasecount, 1)
3022         d.addCallback(self._count_leases, "mutable")
3023         d.addCallback(self._assert_leasecount, 1)
3024
3025         # this CHECK uses an alternate client, which adds a second lease
3026         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3027         d.addCallback(_got_html_good)
3028
3029         d.addCallback(self._count_leases, "one")
3030         d.addCallback(self._assert_leasecount, 2)
3031         d.addCallback(self._count_leases, "two")
3032         d.addCallback(self._assert_leasecount, 1)
3033         d.addCallback(self._count_leases, "mutable")
3034         d.addCallback(self._assert_leasecount, 1)
3035
3036         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3037         d.addCallback(_got_html_good)
3038
3039         d.addCallback(self._count_leases, "one")
3040         d.addCallback(self._assert_leasecount, 2)
3041         d.addCallback(self._count_leases, "two")
3042         d.addCallback(self._assert_leasecount, 1)
3043         d.addCallback(self._count_leases, "mutable")
3044         d.addCallback(self._assert_leasecount, 1)
3045
3046         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3047                       clientnum=1)
3048         d.addCallback(_got_html_good)
3049
3050         d.addCallback(self._count_leases, "one")
3051         d.addCallback(self._assert_leasecount, 2)
3052         d.addCallback(self._count_leases, "two")
3053         d.addCallback(self._assert_leasecount, 1)
3054         d.addCallback(self._count_leases, "mutable")
3055         d.addCallback(self._assert_leasecount, 2)
3056
3057         d.addErrback(self.explain_web_error)
3058         return d
3059
3060     def test_deep_add_lease(self):
3061         self.basedir = "web/Grid/deep_add_lease"
3062         self.set_up_grid(num_clients=2)
3063         c0 = self.g.clients[0]
3064         self.uris = {}
3065         self.fileurls = {}
3066         DATA = "data" * 100
3067         d = c0.create_empty_dirnode()
3068         def _stash_root_and_create_file(n):
3069             self.rootnode = n
3070             self.uris["root"] = n.get_uri()
3071             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3072             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3073         d.addCallback(_stash_root_and_create_file)
3074         def _stash_uri(fn, which):
3075             self.uris[which] = fn.get_uri()
3076         d.addCallback(_stash_uri, "one")
3077         d.addCallback(lambda ign:
3078                       self.rootnode.add_file(u"small",
3079                                              upload.Data("literal",
3080                                                         convergence="")))
3081         d.addCallback(_stash_uri, "small")
3082
3083         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3084         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3085         d.addCallback(_stash_uri, "mutable")
3086
3087         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3088         def _done(res):
3089             units = [simplejson.loads(line)
3090                      for line in res.splitlines()
3091                      if line]
3092             # root, one, small, mutable,   stats
3093             self.failUnlessEqual(len(units), 4+1)
3094         d.addCallback(_done)
3095
3096         d.addCallback(self._count_leases, "root")
3097         d.addCallback(self._assert_leasecount, 1)
3098         d.addCallback(self._count_leases, "one")
3099         d.addCallback(self._assert_leasecount, 1)
3100         d.addCallback(self._count_leases, "mutable")
3101         d.addCallback(self._assert_leasecount, 1)
3102
3103         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3104         d.addCallback(_done)
3105
3106         d.addCallback(self._count_leases, "root")
3107         d.addCallback(self._assert_leasecount, 1)
3108         d.addCallback(self._count_leases, "one")
3109         d.addCallback(self._assert_leasecount, 1)
3110         d.addCallback(self._count_leases, "mutable")
3111         d.addCallback(self._assert_leasecount, 1)
3112
3113         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3114                       clientnum=1)
3115         d.addCallback(_done)
3116
3117         d.addCallback(self._count_leases, "root")
3118         d.addCallback(self._assert_leasecount, 2)
3119         d.addCallback(self._count_leases, "one")
3120         d.addCallback(self._assert_leasecount, 2)
3121         d.addCallback(self._count_leases, "mutable")
3122         d.addCallback(self._assert_leasecount, 2)
3123
3124         d.addErrback(self.explain_web_error)
3125         return d