src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from allmydata import interfaces, uri, webish, storage
  10 from allmydata.immutable import upload, download
  11 from allmydata.web import status, common
  12 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  13 from allmydata.util import fileutil, base32
  14 from allmydata.util.assertutil import precondition
  15 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  16      FakeMutableFileNode, create_chk_filenode, WebErrorMixin
  17 from allmydata.interfaces import IURI, INewDirectoryURI, \
  18      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  19 from allmydata.mutable import servermap, publish, retrieve
  20 import common_util as testutil
  21 from allmydata.test.no_network import GridTestMixin
  22
  23 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  24 # create a webserver that works against them
  25
  26 class FakeIntroducerClient:
  27     def get_all_connectors(self):
  28         return {}
  29     def get_all_connections_for(self, service_name):
  30         return frozenset()
  31     def get_all_peerids(self):
  32         return frozenset()
  33
  34 class FakeClient(service.MultiService):
  35     nodeid = "fake_nodeid"
  36     nickname = "fake_nickname"
  37     basedir = "fake_basedir"
  38     def get_versions(self):
  39         return {'allmydata': "fake",
  40                 'foolscap': "fake",
  41                 'twisted': "fake",
  42                 'zfec': "fake",
  43                 }
  44     introducer_furl = "None"
  45     introducer_client = FakeIntroducerClient()
  46     _all_upload_status = [upload.UploadStatus()]
  47     _all_download_status = [download.DownloadStatus()]
  48     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  49     _all_publish_statuses = [publish.PublishStatus()]
  50     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  51     convergence = "some random string"
  52
  53     def connected_to_introducer(self):
  54         return False
  55
  56     def get_nickname_for_peerid(self, peerid):
  57         return u"John Doe"
  58
  59     def get_permuted_peers(self, service_name, key):
  60         return []
  61
  62     def create_node_from_uri(self, auri):
  63         precondition(isinstance(auri, str), auri)
  64         u = uri.from_string(auri)
  65         if (INewDirectoryURI.providedBy(u)
  66             or IReadonlyNewDirectoryURI.providedBy(u)):
  67             return FakeDirectoryNode(self).init_from_uri(u)
  68         if IFileURI.providedBy(u):
  69             return FakeCHKFileNode(u, self)
  70         assert IMutableFileURI.providedBy(u), u
  71         return FakeMutableFileNode(self).init_from_uri(u)
  72
  73     def create_empty_dirnode(self):
  74         n = FakeDirectoryNode(self)
  75         d = n.create()
  76         d.addCallback(lambda res: n)
  77         return d
  78
  79     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  80     def create_mutable_file(self, contents=""):
  81         n = FakeMutableFileNode(self)
  82         return n.create(contents)
  83
  84     def upload(self, uploadable):
  85         d = uploadable.get_size()
  86         d.addCallback(lambda size: uploadable.read(size))
  87         def _got_data(datav):
  88             data = "".join(datav)
  89             n = create_chk_filenode(self, data)
  90             results = upload.UploadResults()
  91             results.uri = n.get_uri()
  92             return results
  93         d.addCallback(_got_data)
  94         return d
  95
  96     def list_all_upload_statuses(self):
  97         return self._all_upload_status
  98     def list_all_download_statuses(self):
  99         return self._all_download_status
 100     def list_all_mapupdate_statuses(self):
 101         return self._all_mapupdate_statuses
 102     def list_all_publish_statuses(self):
 103         return self._all_publish_statuses
 104     def list_all_retrieve_statuses(self):
 105         return self._all_retrieve_statuses
 106     def list_all_helper_statuses(self):
 107         return []
 108
 109 class MyGetter(client.HTTPPageGetter):
 110     handleStatus_206 = lambda self: self.handleStatus_200()
 111
 112 class HTTPClientHEADFactory(client.HTTPClientFactory):
 113     protocol = MyGetter
 114
 115     def noPage(self, reason):
 116         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 117         # exception when the response to a HEAD request had no body (when in
 118         # fact they are defined to never have a body). This was fixed in
 119         # Twisted-8.0 . To work around this, we catch the
 120         # PartialDownloadError and make it disappear.
 121         if (reason.check(client.PartialDownloadError)
 122             and self.method.upper() == "HEAD"):
 123             self.page("")
 124             return
 125         return client.HTTPClientFactory.noPage(self, reason)
 126
 127 class HTTPClientGETFactory(client.HTTPClientFactory):
 128     protocol = MyGetter
 129
 130 class WebMixin(object):
 131     def setUp(self):
 132         self.s = FakeClient()
 133         self.s.startService()
 134         self.staticdir = self.mktemp()
 135         self.ws = s = webish.WebishServer("0", staticdir=self.staticdir)
 136         s.setServiceParent(self.s)
 137         self.webish_port = port = s.listener._port.getHost().port
 138         self.webish_url = "http://localhost:%d" % port
 139
 140         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 141         d = defer.DeferredList(l)
 142         def _then(res):
 143             self.public_root = res[0][1]
 144             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 145             self.public_url = "/uri/" + self.public_root.get_uri()
 146             self.private_root = res[1][1]
 147
 148             foo = res[2][1]
 149             self._foo_node = foo
 150             self._foo_uri = foo.get_uri()
 151             self._foo_readonly_uri = foo.get_readonly_uri()
 152             self._foo_verifycap = foo.get_verify_cap().to_string()
 153             # NOTE: we ignore the deferred on all set_uri() calls, because we
 154             # know the fake nodes do these synchronously
 155             self.public_root.set_uri(u"foo", foo.get_uri())
 156
 157             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 158             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 159             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 160
 161             foo.set_uri(u"empty", res[3][1].get_uri())
 162             sub_uri = res[4][1].get_uri()
 163             self._sub_uri = sub_uri
 164             foo.set_uri(u"sub", sub_uri)
 165             sub = self.s.create_node_from_uri(sub_uri)
 166
 167             _ign, n, blocking_uri = self.makefile(1)
 168             foo.set_uri(u"blockingfile", blocking_uri)
 169
 170             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 171             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 172             # still think of it as an umlaut
 173             foo.set_uri(unicode_filename, self._bar_txt_uri)
 174
 175             _ign, n, baz_file = self.makefile(2)
 176             self._baz_file_uri = baz_file
 177             sub.set_uri(u"baz.txt", baz_file)
 178
 179             _ign, n, self._bad_file_uri = self.makefile(3)
 180             # this uri should not be downloadable
 181             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 182
 183             rodir = res[5][1]
 184             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 185             rodir.set_uri(u"nor", baz_file)
 186
 187             # public/
 188             # public/foo/
 189             # public/foo/bar.txt
 190             # public/foo/blockingfile
 191             # public/foo/empty/
 192             # public/foo/sub/
 193             # public/foo/sub/baz.txt
 194             # public/reedownlee/
 195             # public/reedownlee/nor
 196             self.NEWFILE_CONTENTS = "newfile contents\n"
 197
 198             return foo.get_metadata_for(u"bar.txt")
 199         d.addCallback(_then)
 200         def _got_metadata(metadata):
 201             self._bar_txt_metadata = metadata
 202         d.addCallback(_got_metadata)
 203         return d
 204
 205     def makefile(self, number):
 206         contents = "contents of file %s\n" % number
 207         n = create_chk_filenode(self.s, contents)
 208         return contents, n, n.get_uri()
 209
 210     def tearDown(self):
 211         return self.s.stopService()
 212
 213     def failUnlessIsBarDotTxt(self, res):
 214         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 215
 216     def failUnlessIsBarJSON(self, res):
 217         data = simplejson.loads(res)
 218         self.failUnless(isinstance(data, list))
 219         self.failUnlessEqual(data[0], u"filenode")
 220         self.failUnless(isinstance(data[1], dict))
 221         self.failIf(data[1]["mutable"])
 222         self.failIf("rw_uri" in data[1]) # immutable
 223         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 224         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 225         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 226
 227     def failUnlessIsFooJSON(self, res):
 228         data = simplejson.loads(res)
 229         self.failUnless(isinstance(data, list))
 230         self.failUnlessEqual(data[0], "dirnode", res)
 231         self.failUnless(isinstance(data[1], dict))
 232         self.failUnless(data[1]["mutable"])
 233         self.failUnless("rw_uri" in data[1]) # mutable
 234         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 235         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 236         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 237
 238         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 239         self.failUnlessEqual(kidnames,
 240                              [u"bar.txt", u"blockingfile", u"empty",
 241                               u"n\u00fc.txt", u"sub"])
 242         kids = dict( [(unicode(name),value)
 243                       for (name,value)
 244                       in data[1]["children"].iteritems()] )
 245         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 246         self.failUnless("metadata" in kids[u"sub"][1])
 247         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 248         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 249         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 250         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 252         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 253                              self._bar_txt_verifycap)
 254         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 255                              self._bar_txt_metadata["ctime"])
 256         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 257                              self._bar_txt_uri)
 258
 259     def GET(self, urlpath, followRedirect=False, return_response=False,
 260             **kwargs):
 261         # if return_response=True, this fires with (data, statuscode,
 262         # respheaders) instead of just data.
 263         assert not isinstance(urlpath, unicode)
 264         url = self.webish_url + urlpath
 265         factory = HTTPClientGETFactory(url, method="GET",
 266                                        followRedirect=followRedirect, **kwargs)
 267         reactor.connectTCP("localhost", self.webish_port, factory)
 268         d = factory.deferred
 269         def _got_data(data):
 270             return (data, factory.status, factory.response_headers)
 271         if return_response:
 272             d.addCallback(_got_data)
 273         return factory.deferred
 274
 275     def HEAD(self, urlpath, return_response=False, **kwargs):
 276         # this requires some surgery, because twisted.web.client doesn't want
 277         # to give us back the response headers.
 278         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 279         reactor.connectTCP("localhost", self.webish_port, factory)
 280         d = factory.deferred
 281         def _got_data(data):
 282             return (data, factory.status, factory.response_headers)
 283         if return_response:
 284             d.addCallback(_got_data)
 285         return factory.deferred
 286
 287     def PUT(self, urlpath, data, **kwargs):
 288         url = self.webish_url + urlpath
 289         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 290
 291     def DELETE(self, urlpath):
 292         url = self.webish_url + urlpath
 293         return client.getPage(url, method="DELETE")
 294
 295     def POST(self, urlpath, followRedirect=False, **fields):
 296         url = self.webish_url + urlpath
 297         sepbase = "boogabooga"
 298         sep = "--" + sepbase
 299         form = []
 300         form.append(sep)
 301         form.append('Content-Disposition: form-data; name="_charset"')
 302         form.append('')
 303         form.append('UTF-8')
 304         form.append(sep)
 305         for name, value in fields.iteritems():
 306             if isinstance(value, tuple):
 307                 filename, value = value
 308                 form.append('Content-Disposition: form-data; name="%s"; '
 309                             'filename="%s"' % (name, filename.encode("utf-8")))
 310             else:
 311                 form.append('Content-Disposition: form-data; name="%s"' % name)
 312             form.append('')
 313             if isinstance(value, unicode):
 314                 value = value.encode("utf-8")
 315             else:
 316                 value = str(value)
 317             assert isinstance(value, str)
 318             form.append(value)
 319             form.append(sep)
 320         form[-1] += "--"
 321         body = "\r\n".join(form) + "\r\n"
 322         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 323                    }
 324         return client.getPage(url, method="POST", postdata=body,
 325                               headers=headers, followRedirect=followRedirect)
 326
 327     def shouldFail(self, res, expected_failure, which,
 328                    substring=None, response_substring=None):
 329         if isinstance(res, failure.Failure):
 330             res.trap(expected_failure)
 331             if substring:
 332                 self.failUnless(substring in str(res),
 333                                 "substring '%s' not in '%s'"
 334                                 % (substring, str(res)))
 335             if response_substring:
 336                 self.failUnless(response_substring in res.value.response,
 337                                 "response substring '%s' not in '%s'"
 338                                 % (response_substring, res.value.response))
 339         else:
 340             self.fail("%s was supposed to raise %s, not get '%s'" %
 341                       (which, expected_failure, res))
 342
 343     def shouldFail2(self, expected_failure, which, substring,
 344                     response_substring,
 345                     callable, *args, **kwargs):
 346         assert substring is None or isinstance(substring, str)
 347         assert response_substring is None or isinstance(response_substring, str)
 348         d = defer.maybeDeferred(callable, *args, **kwargs)
 349         def done(res):
 350             if isinstance(res, failure.Failure):
 351                 res.trap(expected_failure)
 352                 if substring:
 353                     self.failUnless(substring in str(res),
 354                                     "%s: substring '%s' not in '%s'"
 355                                     % (which, substring, str(res)))
 356                 if response_substring:
 357                     self.failUnless(response_substring in res.value.response,
 358                                     "%s: response substring '%s' not in '%s'"
 359                                     % (which,
 360                                        response_substring, res.value.response))
 361             else:
 362                 self.fail("%s was supposed to raise %s, not get '%s'" %
 363                           (which, expected_failure, res))
 364         d.addBoth(done)
 365         return d
 366
 367     def should404(self, res, which):
 368         if isinstance(res, failure.Failure):
 369             res.trap(error.Error)
 370             self.failUnlessEqual(res.value.status, "404")
 371         else:
 372             self.fail("%s was supposed to Error(404), not get '%s'" %
 373                       (which, res))
 374
 375     def shouldHTTPError(self, res, which, code=None, substring=None,
 376                         response_substring=None):
 377         if isinstance(res, failure.Failure):
 378             res.trap(error.Error)
 379             if code is not None:
 380                 self.failUnlessEqual(res.value.status, str(code))
 381             if substring:
 382                 self.failUnless(substring in str(res),
 383                                 "substring '%s' not in '%s'"
 384                                 % (substring, str(res)))
 385             if response_substring:
 386                 self.failUnless(response_substring in res.value.response,
 387                                 "response substring '%s' not in '%s'"
 388                                 % (response_substring, res.value.response))
 389         else:
 390             self.fail("%s was supposed to Error(%s), not get '%s'" %
 391                       (which, code, res))
 392
 393     def shouldHTTPError2(self, which,
 394                          code=None, substring=None, response_substring=None,
 395                          callable=None, *args, **kwargs):
 396         assert substring is None or isinstance(substring, str)
 397         assert callable
 398         d = defer.maybeDeferred(callable, *args, **kwargs)
 399         d.addBoth(self.shouldHTTPError, which,
 400                   code, substring, response_substring)
 401         return d
 402
 403
 404 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 405     def test_create(self):
 406         pass
 407
 408     def test_welcome(self):
 409         d = self.GET("/")
 410         def _check(res):
 411             self.failUnless('Welcome To AllMyData' in res)
 412             self.failUnless('Tahoe' in res)
 413
 414             self.s.basedir = 'web/test_welcome'
 415             fileutil.make_dirs("web/test_welcome")
 416             fileutil.make_dirs("web/test_welcome/private")
 417             return self.GET("/")
 418         d.addCallback(_check)
 419         return d
 420
 421     def test_provisioning(self):
 422         d = self.GET("/provisioning/")
 423         def _check(res):
 424             self.failUnless('Tahoe Provisioning Tool' in res)
 425             fields = {'filled': True,
 426                       "num_users": int(50e3),
 427                       "files_per_user": 1000,
 428                       "space_per_user": int(1e9),
 429                       "sharing_ratio": 1.0,
 430                       "encoding_parameters": "3-of-10-5",
 431                       "num_servers": 30,
 432                       "ownership_mode": "A",
 433                       "download_rate": 100,
 434                       "upload_rate": 10,
 435                       "delete_rate": 10,
 436                       "lease_timer": 7,
 437                       }
 438             return self.POST("/provisioning/", **fields)
 439
 440         d.addCallback(_check)
 441         def _check2(res):
 442             self.failUnless('Tahoe Provisioning Tool' in res)
 443             self.failUnless("Share space consumed: 167.01TB" in res)
 444
 445             fields = {'filled': True,
 446                       "num_users": int(50e6),
 447                       "files_per_user": 1000,
 448                       "space_per_user": int(5e9),
 449                       "sharing_ratio": 1.0,
 450                       "encoding_parameters": "25-of-100-50",
 451                       "num_servers": 30000,
 452                       "ownership_mode": "E",
 453                       "drive_failure_model": "U",
 454                       "drive_size": 1000,
 455                       "download_rate": 1000,
 456                       "upload_rate": 100,
 457                       "delete_rate": 100,
 458                       "lease_timer": 7,
 459                       }
 460             return self.POST("/provisioning/", **fields)
 461         d.addCallback(_check2)
 462         def _check3(res):
 463             self.failUnless("Share space consumed: huge!" in res)
 464             fields = {'filled': True}
 465             return self.POST("/provisioning/", **fields)
 466         d.addCallback(_check3)
 467         def _check4(res):
 468             self.failUnless("Share space consumed:" in res)
 469         d.addCallback(_check4)
 470         return d
 471
 472     def test_reliability_tool(self):
 473         try:
 474             from allmydata import reliability
 475             _hush_pyflakes = reliability
 476         except:
 477             raise unittest.SkipTest("reliability tool requires Numeric")
 478
 479         d = self.GET("/reliability/")
 480         def _check(res):
 481             self.failUnless('Tahoe Reliability Tool' in res)
 482             fields = {'drive_lifetime': "8Y",
 483                       "k": "3",
 484                       "R": "7",
 485                       "N": "10",
 486                       "delta": "100000",
 487                       "check_period": "1M",
 488                       "report_period": "3M",
 489                       "report_span": "5Y",
 490                       }
 491             return self.POST("/reliability/", **fields)
 492
 493         d.addCallback(_check)
 494         def _check2(res):
 495             self.failUnless('Tahoe Reliability Tool' in res)
 496             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 497             self.failUnless(re.search(r, res), res)
 498         d.addCallback(_check2)
 499         return d
 500
 501     def test_status(self):
 502         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 503         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 504         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 505         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 506         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 507         d = self.GET("/status", followRedirect=True)
 508         def _check(res):
 509             self.failUnless('Upload and Download Status' in res, res)
 510             self.failUnless('"down-%d"' % dl_num in res, res)
 511             self.failUnless('"up-%d"' % ul_num in res, res)
 512             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 513             self.failUnless('"publish-%d"' % pub_num in res, res)
 514             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 515         d.addCallback(_check)
 516         d.addCallback(lambda res: self.GET("/status/?t=json"))
 517         def _check_json(res):
 518             data = simplejson.loads(res)
 519             self.failUnless(isinstance(data, dict))
 520             active = data["active"]
 521             # TODO: test more. We need a way to fake an active operation
 522             # here.
 523         d.addCallback(_check_json)
 524
 525         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 526         def _check_dl(res):
 527             self.failUnless("File Download Status" in res, res)
 528         d.addCallback(_check_dl)
 529         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 530         def _check_ul(res):
 531             self.failUnless("File Upload Status" in res, res)
 532         d.addCallback(_check_ul)
 533         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 534         def _check_mapupdate(res):
 535             self.failUnless("Mutable File Servermap Update Status" in res, res)
 536         d.addCallback(_check_mapupdate)
 537         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 538         def _check_publish(res):
 539             self.failUnless("Mutable File Publish Status" in res, res)
 540         d.addCallback(_check_publish)
 541         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 542         def _check_retrieve(res):
 543             self.failUnless("Mutable File Retrieve Status" in res, res)
 544         d.addCallback(_check_retrieve)
 545
 546         return d
 547
 548     def test_status_numbers(self):
 549         drrm = status.DownloadResultsRendererMixin()
 550         self.failUnlessEqual(drrm.render_time(None, None), "")
 551         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 552         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 553         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 554         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 555         self.failUnlessEqual(drrm.render_rate(None, None), "")
 556         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 557         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 558         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 559
 560         urrm = status.UploadResultsRendererMixin()
 561         self.failUnlessEqual(urrm.render_time(None, None), "")
 562         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 563         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 564         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 565         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 566         self.failUnlessEqual(urrm.render_rate(None, None), "")
 567         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 568         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 569         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 570
 571     def test_GET_FILEURL(self):
 572         d = self.GET(self.public_url + "/foo/bar.txt")
 573         d.addCallback(self.failUnlessIsBarDotTxt)
 574         return d
 575
 576     def test_GET_FILEURL_range(self):
 577         headers = {"range": "bytes=1-10"}
 578         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 579                      return_response=True)
 580         def _got((res, status, headers)):
 581             self.failUnlessEqual(int(status), 206)
 582             self.failUnless(headers.has_key("content-range"))
 583             self.failUnlessEqual(headers["content-range"][0],
 584                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 585             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 586         d.addCallback(_got)
 587         return d
 588
 589     def test_GET_FILEURL_partial_range(self):
 590         headers = {"range": "bytes=5-"}
 591         length  = len(self.BAR_CONTENTS)
 592         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 593                      return_response=True)
 594         def _got((res, status, headers)):
 595             self.failUnlessEqual(int(status), 206)
 596             self.failUnless(headers.has_key("content-range"))
 597             self.failUnlessEqual(headers["content-range"][0],
 598                                  "bytes 5-%d/%d" % (length-1, length))
 599             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 600         d.addCallback(_got)
 601         return d
 602
 603     def test_HEAD_FILEURL_range(self):
 604         headers = {"range": "bytes=1-10"}
 605         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 606                      return_response=True)
 607         def _got((res, status, headers)):
 608             self.failUnlessEqual(res, "")
 609             self.failUnlessEqual(int(status), 206)
 610             self.failUnless(headers.has_key("content-range"))
 611             self.failUnlessEqual(headers["content-range"][0],
 612                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 613         d.addCallback(_got)
 614         return d
 615
 616     def test_HEAD_FILEURL_partial_range(self):
 617         headers = {"range": "bytes=5-"}
 618         length  = len(self.BAR_CONTENTS)
 619         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 620                      return_response=True)
 621         def _got((res, status, headers)):
 622             self.failUnlessEqual(int(status), 206)
 623             self.failUnless(headers.has_key("content-range"))
 624             self.failUnlessEqual(headers["content-range"][0],
 625                                  "bytes 5-%d/%d" % (length-1, length))
 626         d.addCallback(_got)
 627         return d
 628
 629     def test_GET_FILEURL_range_bad(self):
 630         headers = {"range": "BOGUS=fizbop-quarnak"}
 631         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 632                              "400 Bad Request",
 633                              "Syntactically invalid http range header",
 634                              self.GET, self.public_url + "/foo/bar.txt",
 635                              headers=headers)
 636         return d
 637
 638     def test_HEAD_FILEURL(self):
 639         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 640         def _got((res, status, headers)):
 641             self.failUnlessEqual(res, "")
 642             self.failUnlessEqual(headers["content-length"][0],
 643                                  str(len(self.BAR_CONTENTS)))
 644             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 645         d.addCallback(_got)
 646         return d
 647
 648     def test_GET_FILEURL_named(self):
 649         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 650         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 651         d = self.GET(base + "/@@name=/blah.txt")
 652         d.addCallback(self.failUnlessIsBarDotTxt)
 653         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 654         d.addCallback(self.failUnlessIsBarDotTxt)
 655         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 656         d.addCallback(self.failUnlessIsBarDotTxt)
 657         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 658         d.addCallback(self.failUnlessIsBarDotTxt)
 659         save_url = base + "?save=true&filename=blah.txt"
 660         d.addCallback(lambda res: self.GET(save_url))
 661         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 662         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 663         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 664         u_url = base + "?save=true&filename=" + u_fn_e
 665         d.addCallback(lambda res: self.GET(u_url))
 666         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 667         return d
 668
 669     def test_PUT_FILEURL_named_bad(self):
 670         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 671         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 672                              "400 Bad Request",
 673                              "/file can only be used with GET or HEAD",
 674                              self.PUT, base + "/@@name=/blah.txt", "")
 675         return d
 676
 677     def test_GET_DIRURL_named_bad(self):
 678         base = "/file/%s" % urllib.quote(self._foo_uri)
 679         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 680                              "400 Bad Request",
 681                              "is not a file-cap",
 682                              self.GET, base + "/@@name=/blah.txt")
 683         return d
 684
 685     def test_GET_slash_file_bad(self):
 686         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 687                              "404 Not Found",
 688                              "/file must be followed by a file-cap and a name",
 689                              self.GET, "/file")
 690         return d
 691
 692     def test_GET_unhandled_URI_named(self):
 693         contents, n, newuri = self.makefile(12)
 694         verifier_cap = n.get_verify_cap().to_string()
 695         base = "/file/%s" % urllib.quote(verifier_cap)
 696         # client.create_node_from_uri() can't handle verify-caps
 697         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 698                              "400 Bad Request",
 699                              "is not a valid file- or directory- cap",
 700                              self.GET, base)
 701         return d
 702
 703     def test_GET_unhandled_URI(self):
 704         contents, n, newuri = self.makefile(12)
 705         verifier_cap = n.get_verify_cap().to_string()
 706         base = "/uri/%s" % urllib.quote(verifier_cap)
 707         # client.create_node_from_uri() can't handle verify-caps
 708         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 709                              "400 Bad Request",
 710                              "is not a valid file- or directory- cap",
 711                              self.GET, base)
 712         return d
 713
 714     def test_GET_FILE_URI(self):
 715         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 716         d = self.GET(base)
 717         d.addCallback(self.failUnlessIsBarDotTxt)
 718         return d
 719
 720     def test_GET_FILE_URI_badchild(self):
 721         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 722         errmsg = "Files have no children, certainly not named 'boguschild'"
 723         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 724                              "400 Bad Request", errmsg,
 725                              self.GET, base)
 726         return d
 727
 728     def test_PUT_FILE_URI_badchild(self):
 729         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 730         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 731         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 732                              "400 Bad Request", errmsg,
 733                              self.PUT, base, "")
 734         return d
 735
 736     def test_GET_FILEURL_save(self):
 737         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 738         # TODO: look at the headers, expect a Content-Disposition: attachment
 739         # header.
 740         d.addCallback(self.failUnlessIsBarDotTxt)
 741         return d
 742
 743     def test_GET_FILEURL_missing(self):
 744         d = self.GET(self.public_url + "/foo/missing")
 745         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 746         return d
 747
 748     def test_PUT_NEWFILEURL(self):
 749         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 750         # TODO: we lose the response code, so we can't check this
 751         #self.failUnlessEqual(responsecode, 201)
 752         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 753         d.addCallback(lambda res:
 754                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 755                                                       self.NEWFILE_CONTENTS))
 756         return d
 757
 758     def test_PUT_NEWFILEURL_range_bad(self):
 759         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 760         target = self.public_url + "/foo/new.txt"
 761         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 762                              "501 Not Implemented",
 763                              "Content-Range in PUT not yet supported",
 764                              # (and certainly not for immutable files)
 765                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 766                              headers=headers)
 767         d.addCallback(lambda res:
 768                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_mutable(self):
 772         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 773                      self.NEWFILE_CONTENTS)
 774         # TODO: we lose the response code, so we can't check this
 775         #self.failUnlessEqual(responsecode, 201)
 776         def _check_uri(res):
 777             u = uri.from_string_mutable_filenode(res)
 778             self.failUnless(u.is_mutable())
 779             self.failIf(u.is_readonly())
 780             return res
 781         d.addCallback(_check_uri)
 782         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 783         d.addCallback(lambda res:
 784                       self.failUnlessMutableChildContentsAre(self._foo_node,
 785                                                              u"new.txt",
 786                                                              self.NEWFILE_CONTENTS))
 787         return d
 788
 789     def test_PUT_NEWFILEURL_mutable_toobig(self):
 790         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 791                              "413 Request Entity Too Large",
 792                              "SDMF is limited to one segment, and 10001 > 10000",
 793                              self.PUT,
 794                              self.public_url + "/foo/new.txt?mutable=true",
 795                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 796         return d
 797
 798     def test_PUT_NEWFILEURL_replace(self):
 799         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 800         # TODO: we lose the response code, so we can't check this
 801         #self.failUnlessEqual(responsecode, 200)
 802         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 803         d.addCallback(lambda res:
 804                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 805                                                       self.NEWFILE_CONTENTS))
 806         return d
 807
 808     def test_PUT_NEWFILEURL_bad_t(self):
 809         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 810                              "PUT to a file: bad t=bogus",
 811                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 812                              "contents")
 813         return d
 814
 815     def test_PUT_NEWFILEURL_no_replace(self):
 816         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 817                      self.NEWFILE_CONTENTS)
 818         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 819                   "409 Conflict",
 820                   "There was already a child by that name, and you asked me "
 821                   "to not replace it")
 822         return d
 823
 824     def test_PUT_NEWFILEURL_mkdirs(self):
 825         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 826         fn = self._foo_node
 827         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 828         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 829         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 830         d.addCallback(lambda res:
 831                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 832                                                       self.NEWFILE_CONTENTS))
 833         return d
 834
 835     def test_PUT_NEWFILEURL_blocked(self):
 836         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 837                      self.NEWFILE_CONTENTS)
 838         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 839                   "409 Conflict",
 840                   "Unable to create directory 'blockingfile': a file was in the way")
 841         return d
 842
 843     def test_DELETE_FILEURL(self):
 844         d = self.DELETE(self.public_url + "/foo/bar.txt")
 845         d.addCallback(lambda res:
 846                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 847         return d
 848
 849     def test_DELETE_FILEURL_missing(self):
 850         d = self.DELETE(self.public_url + "/foo/missing")
 851         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 852         return d
 853
 854     def test_DELETE_FILEURL_missing2(self):
 855         d = self.DELETE(self.public_url + "/missing/missing")
 856         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 857         return d
 858
 859     def test_GET_FILEURL_json(self):
 860         # twisted.web.http.parse_qs ignores any query args without an '=', so
 861         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 862         # instead. This may make it tricky to emulate the S3 interface
 863         # completely.
 864         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 865         d.addCallback(self.failUnlessIsBarJSON)
 866         return d
 867
 868     def test_GET_FILEURL_json_missing(self):
 869         d = self.GET(self.public_url + "/foo/missing?json")
 870         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 871         return d
 872
 873     def test_GET_FILEURL_uri(self):
 874         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 875         def _check(res):
 876             self.failUnlessEqual(res, self._bar_txt_uri)
 877         d.addCallback(_check)
 878         d.addCallback(lambda res:
 879                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 880         def _check2(res):
 881             # for now, for files, uris and readonly-uris are the same
 882             self.failUnlessEqual(res, self._bar_txt_uri)
 883         d.addCallback(_check2)
 884         return d
 885
 886     def test_GET_FILEURL_badtype(self):
 887         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 888                                   "bad t=bogus",
 889                                   self.GET,
 890                                   self.public_url + "/foo/bar.txt?t=bogus")
 891         return d
 892
 893     def test_GET_FILEURL_uri_missing(self):
 894         d = self.GET(self.public_url + "/foo/missing?t=uri")
 895         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 896         return d
 897
 898     def test_GET_DIRURL(self):
 899         # the addSlash means we get a redirect here
 900         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 901         ROOT = "../../.."
 902         d = self.GET(self.public_url + "/foo", followRedirect=True)
 903         def _check(res):
 904             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 905                             in res, res)
 906             # the FILE reference points to a URI, but it should end in bar.txt
 907             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 908                        (ROOT, urllib.quote(self._bar_txt_uri)))
 909             get_bar = "".join([r'<td>',
 910                                r'<a href="%s">bar.txt</a>' % bar_url,
 911                                r'</td>',
 912                                r'\s+<td>FILE</td>',
 913                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 914                                ])
 915             self.failUnless(re.search(get_bar, res), res)
 916             for line in res.split("\n"):
 917                 # find the line that contains the delete button for bar.txt
 918                 if ("form action" in line and
 919                     'value="delete"' in line and
 920                     'value="bar.txt"' in line):
 921                     # the form target should use a relative URL
 922                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 923                     self.failUnless(('action="%s"' % foo_url) in line, line)
 924                     # and the when_done= should too
 925                     #done_url = urllib.quote(???)
 926                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 927                     #                in line, line)
 928                     break
 929             else:
 930                 self.fail("unable to find delete-bar.txt line", res)
 931
 932             # the DIR reference just points to a URI
 933             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 934             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 935                        + r'\s+<td>DIR</td>')
 936             self.failUnless(re.search(get_sub, res), res)
 937         d.addCallback(_check)
 938
 939         # look at a directory which is readonly
 940         d.addCallback(lambda res:
 941                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 942         def _check2(res):
 943             self.failUnless("(readonly)" in res, res)
 944             self.failIf("Upload a file" in res, res)
 945         d.addCallback(_check2)
 946
 947         # and at a directory that contains a readonly directory
 948         d.addCallback(lambda res:
 949                       self.GET(self.public_url, followRedirect=True))
 950         def _check3(res):
 951             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 952                                       '</td>\s+<td>DIR-RO</td>', res))
 953         d.addCallback(_check3)
 954
 955         return d
 956
 957     def test_GET_DIRURL_badtype(self):
 958         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 959                                   400, "Bad Request",
 960                                   "bad t=bogus",
 961                                   self.GET,
 962                                   self.public_url + "/foo?t=bogus")
 963         return d
 964
 965     def test_GET_DIRURL_json(self):
 966         d = self.GET(self.public_url + "/foo?t=json")
 967         d.addCallback(self.failUnlessIsFooJSON)
 968         return d
 969
 970
 971     def test_POST_DIRURL_manifest_no_ophandle(self):
 972         d = self.shouldFail2(error.Error,
 973                              "test_POST_DIRURL_manifest_no_ophandle",
 974                              "400 Bad Request",
 975                              "slow operation requires ophandle=",
 976                              self.POST, self.public_url, t="start-manifest")
 977         return d
 978
 979     def test_POST_DIRURL_manifest(self):
 980         d = defer.succeed(None)
 981         def getman(ignored, output):
 982             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 983                           followRedirect=True)
 984             d.addCallback(self.wait_for_operation, "125")
 985             d.addCallback(self.get_operation_results, "125", output)
 986             return d
 987         d.addCallback(getman, None)
 988         def _got_html(manifest):
 989             self.failUnless("Manifest of SI=" in manifest)
 990             self.failUnless("<td>sub</td>" in manifest)
 991             self.failUnless(self._sub_uri in manifest)
 992             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 993         d.addCallback(_got_html)
 994
 995         # both t=status and unadorned GET should be identical
 996         d.addCallback(lambda res: self.GET("/operations/125"))
 997         d.addCallback(_got_html)
 998
 999         d.addCallback(getman, "html")
1000         d.addCallback(_got_html)
1001         d.addCallback(getman, "text")
1002         def _got_text(manifest):
1003             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1004             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1005         d.addCallback(_got_text)
1006         d.addCallback(getman, "JSON")
1007         def _got_json(res):
1008             data = res["manifest"]
1009             got = {}
1010             for (path_list, cap) in data:
1011                 got[tuple(path_list)] = cap
1012             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1013             self.failUnless((u"sub",u"baz.txt") in got)
1014             self.failUnless("finished" in res)
1015             self.failUnless("origin" in res)
1016             self.failUnless("storage-index" in res)
1017             self.failUnless("verifycaps" in res)
1018             self.failUnless("stats" in res)
1019         d.addCallback(_got_json)
1020         return d
1021
1022     def test_POST_DIRURL_deepsize_no_ophandle(self):
1023         d = self.shouldFail2(error.Error,
1024                              "test_POST_DIRURL_deepsize_no_ophandle",
1025                              "400 Bad Request",
1026                              "slow operation requires ophandle=",
1027                              self.POST, self.public_url, t="start-deep-size")
1028         return d
1029
1030     def test_POST_DIRURL_deepsize(self):
1031         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1032                       followRedirect=True)
1033         d.addCallback(self.wait_for_operation, "126")
1034         d.addCallback(self.get_operation_results, "126", "json")
1035         def _got_json(data):
1036             self.failUnlessEqual(data["finished"], True)
1037             size = data["size"]
1038             self.failUnless(size > 1000)
1039         d.addCallback(_got_json)
1040         d.addCallback(self.get_operation_results, "126", "text")
1041         def _got_text(res):
1042             mo = re.search(r'^size: (\d+)$', res, re.M)
1043             self.failUnless(mo, res)
1044             size = int(mo.group(1))
1045             # with directories, the size varies.
1046             self.failUnless(size > 1000)
1047         d.addCallback(_got_text)
1048         return d
1049
1050     def test_POST_DIRURL_deepstats_no_ophandle(self):
1051         d = self.shouldFail2(error.Error,
1052                              "test_POST_DIRURL_deepstats_no_ophandle",
1053                              "400 Bad Request",
1054                              "slow operation requires ophandle=",
1055                              self.POST, self.public_url, t="start-deep-stats")
1056         return d
1057
1058     def test_POST_DIRURL_deepstats(self):
1059         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1060                       followRedirect=True)
1061         d.addCallback(self.wait_for_operation, "127")
1062         d.addCallback(self.get_operation_results, "127", "json")
1063         def _got_json(stats):
1064             expected = {"count-immutable-files": 3,
1065                         "count-mutable-files": 0,
1066                         "count-literal-files": 0,
1067                         "count-files": 3,
1068                         "count-directories": 3,
1069                         "size-immutable-files": 57,
1070                         "size-literal-files": 0,
1071                         #"size-directories": 1912, # varies
1072                         #"largest-directory": 1590,
1073                         "largest-directory-children": 5,
1074                         "largest-immutable-file": 19,
1075                         }
1076             for k,v in expected.iteritems():
1077                 self.failUnlessEqual(stats[k], v,
1078                                      "stats[%s] was %s, not %s" %
1079                                      (k, stats[k], v))
1080             self.failUnlessEqual(stats["size-files-histogram"],
1081                                  [ [11, 31, 3] ])
1082         d.addCallback(_got_json)
1083         return d
1084
1085     def test_POST_DIRURL_stream_manifest(self):
1086         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1087         def _check(res):
1088             self.failUnless(res.endswith("\n"))
1089             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1090             self.failUnlessEqual(len(units), 7)
1091             self.failUnlessEqual(units[-1]["type"], "stats")
1092             first = units[0]
1093             self.failUnlessEqual(first["path"], [])
1094             self.failUnlessEqual(first["cap"], self._foo_uri)
1095             self.failUnlessEqual(first["type"], "directory")
1096             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1097             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1098             self.failIfEqual(baz["storage-index"], None)
1099             self.failIfEqual(baz["verifycap"], None)
1100             self.failIfEqual(baz["repaircap"], None)
1101             return
1102         d.addCallback(_check)
1103         return d
1104
1105     def test_GET_DIRURL_uri(self):
1106         d = self.GET(self.public_url + "/foo?t=uri")
1107         def _check(res):
1108             self.failUnlessEqual(res, self._foo_uri)
1109         d.addCallback(_check)
1110         return d
1111
1112     def test_GET_DIRURL_readonly_uri(self):
1113         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1114         def _check(res):
1115             self.failUnlessEqual(res, self._foo_readonly_uri)
1116         d.addCallback(_check)
1117         return d
1118
1119     def test_PUT_NEWDIRURL(self):
1120         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1121         d.addCallback(lambda res:
1122                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1123         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1124         d.addCallback(self.failUnlessNodeKeysAre, [])
1125         return d
1126
1127     def test_PUT_NEWDIRURL_exists(self):
1128         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1129         d.addCallback(lambda res:
1130                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1131         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1132         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1133         return d
1134
1135     def test_PUT_NEWDIRURL_blocked(self):
1136         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1137                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1138                              self.PUT,
1139                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1140         d.addCallback(lambda res:
1141                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1142         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1143         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1144         return d
1145
1146     def test_PUT_NEWDIRURL_mkdir_p(self):
1147         d = defer.succeed(None)
1148         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1149         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1150         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1151         def mkdir_p(mkpnode):
1152             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1153             d = self.POST(url)
1154             def made_subsub(ssuri):
1155                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1156                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1157                 d = self.POST(url)
1158                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1159                 return d
1160             d.addCallback(made_subsub)
1161             return d
1162         d.addCallback(mkdir_p)
1163         return d
1164
1165     def test_PUT_NEWDIRURL_mkdirs(self):
1166         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1167         d.addCallback(lambda res:
1168                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1169         d.addCallback(lambda res:
1170                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1171         d.addCallback(lambda res:
1172                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1173         d.addCallback(self.failUnlessNodeKeysAre, [])
1174         return d
1175
1176     def test_DELETE_DIRURL(self):
1177         d = self.DELETE(self.public_url + "/foo")
1178         d.addCallback(lambda res:
1179                       self.failIfNodeHasChild(self.public_root, u"foo"))
1180         return d
1181
1182     def test_DELETE_DIRURL_missing(self):
1183         d = self.DELETE(self.public_url + "/foo/missing")
1184         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1185         d.addCallback(lambda res:
1186                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1187         return d
1188
1189     def test_DELETE_DIRURL_missing2(self):
1190         d = self.DELETE(self.public_url + "/missing")
1191         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1192         return d
1193
1194     def dump_root(self):
1195         print "NODEWALK"
1196         w = webish.DirnodeWalkerMixin()
1197         def visitor(childpath, childnode, metadata):
1198             print childpath
1199         d = w.walk(self.public_root, visitor)
1200         return d
1201
1202     def failUnlessNodeKeysAre(self, node, expected_keys):
1203         for k in expected_keys:
1204             assert isinstance(k, unicode)
1205         d = node.list()
1206         def _check(children):
1207             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1208         d.addCallback(_check)
1209         return d
1210     def failUnlessNodeHasChild(self, node, name):
1211         assert isinstance(name, unicode)
1212         d = node.list()
1213         def _check(children):
1214             self.failUnless(name in children)
1215         d.addCallback(_check)
1216         return d
1217     def failIfNodeHasChild(self, node, name):
1218         assert isinstance(name, unicode)
1219         d = node.list()
1220         def _check(children):
1221             self.failIf(name in children)
1222         d.addCallback(_check)
1223         return d
1224
1225     def failUnlessChildContentsAre(self, node, name, expected_contents):
1226         assert isinstance(name, unicode)
1227         d = node.get_child_at_path(name)
1228         d.addCallback(lambda node: node.download_to_data())
1229         def _check(contents):
1230             self.failUnlessEqual(contents, expected_contents)
1231         d.addCallback(_check)
1232         return d
1233
1234     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1235         assert isinstance(name, unicode)
1236         d = node.get_child_at_path(name)
1237         d.addCallback(lambda node: node.download_best_version())
1238         def _check(contents):
1239             self.failUnlessEqual(contents, expected_contents)
1240         d.addCallback(_check)
1241         return d
1242
1243     def failUnlessChildURIIs(self, node, name, expected_uri):
1244         assert isinstance(name, unicode)
1245         d = node.get_child_at_path(name)
1246         def _check(child):
1247             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1248         d.addCallback(_check)
1249         return d
1250
1251     def failUnlessURIMatchesChild(self, got_uri, node, name):
1252         assert isinstance(name, unicode)
1253         d = node.get_child_at_path(name)
1254         def _check(child):
1255             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1256         d.addCallback(_check)
1257         return d
1258
1259     def failUnlessCHKURIHasContents(self, got_uri, contents):
1260         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1261
1262     def test_POST_upload(self):
1263         d = self.POST(self.public_url + "/foo", t="upload",
1264                       file=("new.txt", self.NEWFILE_CONTENTS))
1265         fn = self._foo_node
1266         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1267         d.addCallback(lambda res:
1268                       self.failUnlessChildContentsAre(fn, u"new.txt",
1269                                                       self.NEWFILE_CONTENTS))
1270         return d
1271
1272     def test_POST_upload_unicode(self):
1273         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1274         d = self.POST(self.public_url + "/foo", t="upload",
1275                       file=(filename, self.NEWFILE_CONTENTS))
1276         fn = self._foo_node
1277         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1278         d.addCallback(lambda res:
1279                       self.failUnlessChildContentsAre(fn, filename,
1280                                                       self.NEWFILE_CONTENTS))
1281         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1282         d.addCallback(lambda res: self.GET(target_url))
1283         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1284                                                             self.NEWFILE_CONTENTS,
1285                                                             contents))
1286         return d
1287
1288     def test_POST_upload_unicode_named(self):
1289         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1290         d = self.POST(self.public_url + "/foo", t="upload",
1291                       name=filename,
1292                       file=("overridden", self.NEWFILE_CONTENTS))
1293         fn = self._foo_node
1294         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1295         d.addCallback(lambda res:
1296                       self.failUnlessChildContentsAre(fn, filename,
1297                                                       self.NEWFILE_CONTENTS))
1298         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1299         d.addCallback(lambda res: self.GET(target_url))
1300         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1301                                                             self.NEWFILE_CONTENTS,
1302                                                             contents))
1303         return d
1304
1305     def test_POST_upload_no_link(self):
1306         d = self.POST("/uri", t="upload",
1307                       file=("new.txt", self.NEWFILE_CONTENTS))
1308         def _check_upload_results(page):
1309             # this should be a page which describes the results of the upload
1310             # that just finished.
1311             self.failUnless("Upload Results:" in page)
1312             self.failUnless("URI:" in page)
1313             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1314             mo = uri_re.search(page)
1315             self.failUnless(mo, page)
1316             new_uri = mo.group(1)
1317             return new_uri
1318         d.addCallback(_check_upload_results)
1319         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1320         return d
1321
1322     def test_POST_upload_no_link_whendone(self):
1323         d = self.POST("/uri", t="upload", when_done="/",
1324                       file=("new.txt", self.NEWFILE_CONTENTS))
1325         d.addBoth(self.shouldRedirect, "/")
1326         return d
1327
1328     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1329         d = defer.maybeDeferred(callable, *args, **kwargs)
1330         def done(res):
1331             if isinstance(res, failure.Failure):
1332                 res.trap(error.PageRedirect)
1333                 statuscode = res.value.status
1334                 target = res.value.location
1335                 return checker(statuscode, target)
1336             self.fail("%s: callable was supposed to redirect, not return '%s'"
1337                       % (which, res))
1338         d.addBoth(done)
1339         return d
1340
1341     def test_POST_upload_no_link_whendone_results(self):
1342         def check(statuscode, target):
1343             self.failUnlessEqual(statuscode, str(http.FOUND))
1344             self.failUnless(target.startswith(self.webish_url), target)
1345             return client.getPage(target, method="GET")
1346         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1347                                  check,
1348                                  self.POST, "/uri", t="upload",
1349                                  when_done="/uri/%(uri)s",
1350                                  file=("new.txt", self.NEWFILE_CONTENTS))
1351         d.addCallback(lambda res:
1352                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1353         return d
1354
1355     def test_POST_upload_no_link_mutable(self):
1356         d = self.POST("/uri", t="upload", mutable="true",
1357                       file=("new.txt", self.NEWFILE_CONTENTS))
1358         def _check(new_uri):
1359             new_uri = new_uri.strip()
1360             self.new_uri = new_uri
1361             u = IURI(new_uri)
1362             self.failUnless(IMutableFileURI.providedBy(u))
1363             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1364             n = self.s.create_node_from_uri(new_uri)
1365             return n.download_best_version()
1366         d.addCallback(_check)
1367         def _check2(data):
1368             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1369             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1370         d.addCallback(_check2)
1371         def _check3(data):
1372             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1373             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1374         d.addCallback(_check3)
1375         def _check4(data):
1376             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1377         d.addCallback(_check4)
1378         return d
1379
1380     def test_POST_upload_no_link_mutable_toobig(self):
1381         d = self.shouldFail2(error.Error,
1382                              "test_POST_upload_no_link_mutable_toobig",
1383                              "413 Request Entity Too Large",
1384                              "SDMF is limited to one segment, and 10001 > 10000",
1385                              self.POST,
1386                              "/uri", t="upload", mutable="true",
1387                              file=("new.txt",
1388                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1389         return d
1390
1391     def test_POST_upload_mutable(self):
1392         # this creates a mutable file
1393         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1394                       file=("new.txt", self.NEWFILE_CONTENTS))
1395         fn = self._foo_node
1396         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1397         d.addCallback(lambda res:
1398                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1399                                                              self.NEWFILE_CONTENTS))
1400         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1401         def _got(newnode):
1402             self.failUnless(IMutableFileNode.providedBy(newnode))
1403             self.failUnless(newnode.is_mutable())
1404             self.failIf(newnode.is_readonly())
1405             self._mutable_node = newnode
1406             self._mutable_uri = newnode.get_uri()
1407         d.addCallback(_got)
1408
1409         # now upload it again and make sure that the URI doesn't change
1410         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1411         d.addCallback(lambda res:
1412                       self.POST(self.public_url + "/foo", t="upload",
1413                                 mutable="true",
1414                                 file=("new.txt", NEWER_CONTENTS)))
1415         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1416         d.addCallback(lambda res:
1417                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1418                                                              NEWER_CONTENTS))
1419         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1420         def _got2(newnode):
1421             self.failUnless(IMutableFileNode.providedBy(newnode))
1422             self.failUnless(newnode.is_mutable())
1423             self.failIf(newnode.is_readonly())
1424             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1425         d.addCallback(_got2)
1426
1427         # upload a second time, using PUT instead of POST
1428         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1429         d.addCallback(lambda res:
1430                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1431         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1432         d.addCallback(lambda res:
1433                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1434                                                              NEW2_CONTENTS))
1435
1436         # finally list the directory, since mutable files are displayed
1437         # slightly differently
1438
1439         d.addCallback(lambda res:
1440                       self.GET(self.public_url + "/foo/",
1441                                followRedirect=True))
1442         def _check_page(res):
1443             # TODO: assert more about the contents
1444             self.failUnless("SSK" in res)
1445             return res
1446         d.addCallback(_check_page)
1447
1448         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1449         def _got3(newnode):
1450             self.failUnless(IMutableFileNode.providedBy(newnode))
1451             self.failUnless(newnode.is_mutable())
1452             self.failIf(newnode.is_readonly())
1453             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1454         d.addCallback(_got3)
1455
1456         # look at the JSON form of the enclosing directory
1457         d.addCallback(lambda res:
1458                       self.GET(self.public_url + "/foo/?t=json",
1459                                followRedirect=True))
1460         def _check_page_json(res):
1461             parsed = simplejson.loads(res)
1462             self.failUnlessEqual(parsed[0], "dirnode")
1463             children = dict( [(unicode(name),value)
1464                               for (name,value)
1465                               in parsed[1]["children"].iteritems()] )
1466             self.failUnless("new.txt" in children)
1467             new_json = children["new.txt"]
1468             self.failUnlessEqual(new_json[0], "filenode")
1469             self.failUnless(new_json[1]["mutable"])
1470             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1471             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1472             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1473         d.addCallback(_check_page_json)
1474
1475         # and the JSON form of the file
1476         d.addCallback(lambda res:
1477                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1478         def _check_file_json(res):
1479             parsed = simplejson.loads(res)
1480             self.failUnlessEqual(parsed[0], "filenode")
1481             self.failUnless(parsed[1]["mutable"])
1482             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1483             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1484             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1485         d.addCallback(_check_file_json)
1486
1487         # and look at t=uri and t=readonly-uri
1488         d.addCallback(lambda res:
1489                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1490         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1491         d.addCallback(lambda res:
1492                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1493         def _check_ro_uri(res):
1494             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1495             self.failUnlessEqual(res, ro_uri)
1496         d.addCallback(_check_ro_uri)
1497
1498         # make sure we can get to it from /uri/URI
1499         d.addCallback(lambda res:
1500                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1501         d.addCallback(lambda res:
1502                       self.failUnlessEqual(res, NEW2_CONTENTS))
1503
1504         # and that HEAD computes the size correctly
1505         d.addCallback(lambda res:
1506                       self.HEAD(self.public_url + "/foo/new.txt",
1507                                 return_response=True))
1508         def _got_headers((res, status, headers)):
1509             self.failUnlessEqual(res, "")
1510             self.failUnlessEqual(headers["content-length"][0],
1511                                  str(len(NEW2_CONTENTS)))
1512             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1513         d.addCallback(_got_headers)
1514
1515         # make sure that size errors are displayed correctly for overwrite
1516         d.addCallback(lambda res:
1517                       self.shouldFail2(error.Error,
1518                                        "test_POST_upload_mutable-toobig",
1519                                        "413 Request Entity Too Large",
1520                                        "SDMF is limited to one segment, and 10001 > 10000",
1521                                        self.POST,
1522                                        self.public_url + "/foo", t="upload",
1523                                        mutable="true",
1524                                        file=("new.txt",
1525                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1526                                        ))
1527
1528         d.addErrback(self.dump_error)
1529         return d
1530
1531     def test_POST_upload_mutable_toobig(self):
1532         d = self.shouldFail2(error.Error,
1533                              "test_POST_upload_no_link_mutable_toobig",
1534                              "413 Request Entity Too Large",
1535                              "SDMF is limited to one segment, and 10001 > 10000",
1536                              self.POST,
1537                              self.public_url + "/foo",
1538                              t="upload", mutable="true",
1539                              file=("new.txt",
1540                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1541         return d
1542
1543     def dump_error(self, f):
1544         # if the web server returns an error code (like 400 Bad Request),
1545         # web.client.getPage puts the HTTP response body into the .response
1546         # attribute of the exception object that it gives back. It does not
1547         # appear in the Failure's repr(), so the ERROR that trial displays
1548         # will be rather terse and unhelpful. addErrback this method to the
1549         # end of your chain to get more information out of these errors.
1550         if f.check(error.Error):
1551             print "web.error.Error:"
1552             print f
1553             print f.value.response
1554         return f
1555
1556     def test_POST_upload_replace(self):
1557         d = self.POST(self.public_url + "/foo", t="upload",
1558                       file=("bar.txt", self.NEWFILE_CONTENTS))
1559         fn = self._foo_node
1560         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1561         d.addCallback(lambda res:
1562                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1563                                                       self.NEWFILE_CONTENTS))
1564         return d
1565
1566     def test_POST_upload_no_replace_ok(self):
1567         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1568                       file=("new.txt", self.NEWFILE_CONTENTS))
1569         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1570         d.addCallback(lambda res: self.failUnlessEqual(res,
1571                                                        self.NEWFILE_CONTENTS))
1572         return d
1573
1574     def test_POST_upload_no_replace_queryarg(self):
1575         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1576                       file=("bar.txt", self.NEWFILE_CONTENTS))
1577         d.addBoth(self.shouldFail, error.Error,
1578                   "POST_upload_no_replace_queryarg",
1579                   "409 Conflict",
1580                   "There was already a child by that name, and you asked me "
1581                   "to not replace it")
1582         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1583         d.addCallback(self.failUnlessIsBarDotTxt)
1584         return d
1585
1586     def test_POST_upload_no_replace_field(self):
1587         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1588                       file=("bar.txt", self.NEWFILE_CONTENTS))
1589         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1590                   "409 Conflict",
1591                   "There was already a child by that name, and you asked me "
1592                   "to not replace it")
1593         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1594         d.addCallback(self.failUnlessIsBarDotTxt)
1595         return d
1596
1597     def test_POST_upload_whendone(self):
1598         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1599                       file=("new.txt", self.NEWFILE_CONTENTS))
1600         d.addBoth(self.shouldRedirect, "/THERE")
1601         fn = self._foo_node
1602         d.addCallback(lambda res:
1603                       self.failUnlessChildContentsAre(fn, u"new.txt",
1604                                                       self.NEWFILE_CONTENTS))
1605         return d
1606
1607     def test_POST_upload_named(self):
1608         fn = self._foo_node
1609         d = self.POST(self.public_url + "/foo", t="upload",
1610                       name="new.txt", file=self.NEWFILE_CONTENTS)
1611         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1612         d.addCallback(lambda res:
1613                       self.failUnlessChildContentsAre(fn, u"new.txt",
1614                                                       self.NEWFILE_CONTENTS))
1615         return d
1616
1617     def test_POST_upload_named_badfilename(self):
1618         d = self.POST(self.public_url + "/foo", t="upload",
1619                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1620         d.addBoth(self.shouldFail, error.Error,
1621                   "test_POST_upload_named_badfilename",
1622                   "400 Bad Request",
1623                   "name= may not contain a slash",
1624                   )
1625         # make sure that nothing was added
1626         d.addCallback(lambda res:
1627                       self.failUnlessNodeKeysAre(self._foo_node,
1628                                                  [u"bar.txt", u"blockingfile",
1629                                                   u"empty", u"n\u00fc.txt",
1630                                                   u"sub"]))
1631         return d
1632
1633     def test_POST_FILEURL_check(self):
1634         bar_url = self.public_url + "/foo/bar.txt"
1635         d = self.POST(bar_url, t="check")
1636         def _check(res):
1637             self.failUnless("Healthy :" in res)
1638         d.addCallback(_check)
1639         redir_url = "http://allmydata.org/TARGET"
1640         def _check2(statuscode, target):
1641             self.failUnlessEqual(statuscode, str(http.FOUND))
1642             self.failUnlessEqual(target, redir_url)
1643         d.addCallback(lambda res:
1644                       self.shouldRedirect2("test_POST_FILEURL_check",
1645                                            _check2,
1646                                            self.POST, bar_url,
1647                                            t="check",
1648                                            when_done=redir_url))
1649         d.addCallback(lambda res:
1650                       self.POST(bar_url, t="check", return_to=redir_url))
1651         def _check3(res):
1652             self.failUnless("Healthy :" in res)
1653             self.failUnless("Return to parent directory" in res)
1654             self.failUnless(redir_url in res)
1655         d.addCallback(_check3)
1656
1657         d.addCallback(lambda res:
1658                       self.POST(bar_url, t="check", output="JSON"))
1659         def _check_json(res):
1660             data = simplejson.loads(res)
1661             self.failUnless("storage-index" in data)
1662             self.failUnless(data["results"]["healthy"])
1663         d.addCallback(_check_json)
1664
1665         return d
1666
1667     def test_POST_FILEURL_check_and_repair(self):
1668         bar_url = self.public_url + "/foo/bar.txt"
1669         d = self.POST(bar_url, t="check", repair="true")
1670         def _check(res):
1671             self.failUnless("Healthy :" in res)
1672         d.addCallback(_check)
1673         redir_url = "http://allmydata.org/TARGET"
1674         def _check2(statuscode, target):
1675             self.failUnlessEqual(statuscode, str(http.FOUND))
1676             self.failUnlessEqual(target, redir_url)
1677         d.addCallback(lambda res:
1678                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1679                                            _check2,
1680                                            self.POST, bar_url,
1681                                            t="check", repair="true",
1682                                            when_done=redir_url))
1683         d.addCallback(lambda res:
1684                       self.POST(bar_url, t="check", return_to=redir_url))
1685         def _check3(res):
1686             self.failUnless("Healthy :" in res)
1687             self.failUnless("Return to parent directory" in res)
1688             self.failUnless(redir_url in res)
1689         d.addCallback(_check3)
1690         return d
1691
1692     def test_POST_DIRURL_check(self):
1693         foo_url = self.public_url + "/foo/"
1694         d = self.POST(foo_url, t="check")
1695         def _check(res):
1696             self.failUnless("Healthy :" in res, res)
1697         d.addCallback(_check)
1698         redir_url = "http://allmydata.org/TARGET"
1699         def _check2(statuscode, target):
1700             self.failUnlessEqual(statuscode, str(http.FOUND))
1701             self.failUnlessEqual(target, redir_url)
1702         d.addCallback(lambda res:
1703                       self.shouldRedirect2("test_POST_DIRURL_check",
1704                                            _check2,
1705                                            self.POST, foo_url,
1706                                            t="check",
1707                                            when_done=redir_url))
1708         d.addCallback(lambda res:
1709                       self.POST(foo_url, t="check", return_to=redir_url))
1710         def _check3(res):
1711             self.failUnless("Healthy :" in res, res)
1712             self.failUnless("Return to parent directory" in res)
1713             self.failUnless(redir_url in res)
1714         d.addCallback(_check3)
1715
1716         d.addCallback(lambda res:
1717                       self.POST(foo_url, t="check", output="JSON"))
1718         def _check_json(res):
1719             data = simplejson.loads(res)
1720             self.failUnless("storage-index" in data)
1721             self.failUnless(data["results"]["healthy"])
1722         d.addCallback(_check_json)
1723
1724         return d
1725
1726     def test_POST_DIRURL_check_and_repair(self):
1727         foo_url = self.public_url + "/foo/"
1728         d = self.POST(foo_url, t="check", repair="true")
1729         def _check(res):
1730             self.failUnless("Healthy :" in res, res)
1731         d.addCallback(_check)
1732         redir_url = "http://allmydata.org/TARGET"
1733         def _check2(statuscode, target):
1734             self.failUnlessEqual(statuscode, str(http.FOUND))
1735             self.failUnlessEqual(target, redir_url)
1736         d.addCallback(lambda res:
1737                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1738                                            _check2,
1739                                            self.POST, foo_url,
1740                                            t="check", repair="true",
1741                                            when_done=redir_url))
1742         d.addCallback(lambda res:
1743                       self.POST(foo_url, t="check", return_to=redir_url))
1744         def _check3(res):
1745             self.failUnless("Healthy :" in res)
1746             self.failUnless("Return to parent directory" in res)
1747             self.failUnless(redir_url in res)
1748         d.addCallback(_check3)
1749         return d
1750
1751     def wait_for_operation(self, ignored, ophandle):
1752         url = "/operations/" + ophandle
1753         url += "?t=status&output=JSON"
1754         d = self.GET(url)
1755         def _got(res):
1756             data = simplejson.loads(res)
1757             if not data["finished"]:
1758                 d = self.stall(delay=1.0)
1759                 d.addCallback(self.wait_for_operation, ophandle)
1760                 return d
1761             return data
1762         d.addCallback(_got)
1763         return d
1764
1765     def get_operation_results(self, ignored, ophandle, output=None):
1766         url = "/operations/" + ophandle
1767         url += "?t=status"
1768         if output:
1769             url += "&output=" + output
1770         d = self.GET(url)
1771         def _got(res):
1772             if output and output.lower() == "json":
1773                 return simplejson.loads(res)
1774             return res
1775         d.addCallback(_got)
1776         return d
1777
1778     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1779         d = self.shouldFail2(error.Error,
1780                              "test_POST_DIRURL_deepcheck_no_ophandle",
1781                              "400 Bad Request",
1782                              "slow operation requires ophandle=",
1783                              self.POST, self.public_url, t="start-deep-check")
1784         return d
1785
1786     def test_POST_DIRURL_deepcheck(self):
1787         def _check_redirect(statuscode, target):
1788             self.failUnlessEqual(statuscode, str(http.FOUND))
1789             self.failUnless(target.endswith("/operations/123"))
1790         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1791                                  self.POST, self.public_url,
1792                                  t="start-deep-check", ophandle="123")
1793         d.addCallback(self.wait_for_operation, "123")
1794         def _check_json(data):
1795             self.failUnlessEqual(data["finished"], True)
1796             self.failUnlessEqual(data["count-objects-checked"], 8)
1797             self.failUnlessEqual(data["count-objects-healthy"], 8)
1798         d.addCallback(_check_json)
1799         d.addCallback(self.get_operation_results, "123", "html")
1800         def _check_html(res):
1801             self.failUnless("Objects Checked: <span>8</span>" in res)
1802             self.failUnless("Objects Healthy: <span>8</span>" in res)
1803         d.addCallback(_check_html)
1804
1805         d.addCallback(lambda res:
1806                       self.GET("/operations/123/"))
1807         d.addCallback(_check_html) # should be the same as without the slash
1808
1809         d.addCallback(lambda res:
1810                       self.shouldFail2(error.Error, "one", "404 Not Found",
1811                                        "No detailed results for SI bogus",
1812                                        self.GET, "/operations/123/bogus"))
1813
1814         foo_si = self._foo_node.get_storage_index()
1815         foo_si_s = base32.b2a(foo_si)
1816         d.addCallback(lambda res:
1817                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1818         def _check_foo_json(res):
1819             data = simplejson.loads(res)
1820             self.failUnlessEqual(data["storage-index"], foo_si_s)
1821             self.failUnless(data["results"]["healthy"])
1822         d.addCallback(_check_foo_json)
1823         return d
1824
1825     def test_POST_DIRURL_deepcheck_and_repair(self):
1826         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1827                       ophandle="124", output="json", followRedirect=True)
1828         d.addCallback(self.wait_for_operation, "124")
1829         def _check_json(data):
1830             self.failUnlessEqual(data["finished"], True)
1831             self.failUnlessEqual(data["count-objects-checked"], 8)
1832             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1833             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1834             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1835             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1836             self.failUnlessEqual(data["count-repairs-successful"], 0)
1837             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1838             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1839             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1840             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1841         d.addCallback(_check_json)
1842         d.addCallback(self.get_operation_results, "124", "html")
1843         def _check_html(res):
1844             self.failUnless("Objects Checked: <span>8</span>" in res)
1845
1846             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1847             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1848             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1849
1850             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1851             self.failUnless("Repairs Successful: <span>0</span>" in res)
1852             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1853
1854             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1855             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1856             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1857         d.addCallback(_check_html)
1858         return d
1859
1860     def test_POST_FILEURL_bad_t(self):
1861         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1862                              "POST to file: bad t=bogus",
1863                              self.POST, self.public_url + "/foo/bar.txt",
1864                              t="bogus")
1865         return d
1866
1867     def test_POST_mkdir(self): # return value?
1868         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1869         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1870         d.addCallback(self.failUnlessNodeKeysAre, [])
1871         return d
1872
1873     def test_POST_mkdir_2(self):
1874         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1875         d.addCallback(lambda res:
1876                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1877         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1878         d.addCallback(self.failUnlessNodeKeysAre, [])
1879         return d
1880
1881     def test_POST_mkdirs_2(self):
1882         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1883         d.addCallback(lambda res:
1884                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1885         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1886         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1887         d.addCallback(self.failUnlessNodeKeysAre, [])
1888         return d
1889
1890     def test_POST_mkdir_no_parentdir_noredirect(self):
1891         d = self.POST("/uri?t=mkdir")
1892         def _after_mkdir(res):
1893             uri.NewDirectoryURI.init_from_string(res)
1894         d.addCallback(_after_mkdir)
1895         return d
1896
1897     def test_POST_mkdir_no_parentdir_redirect(self):
1898         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1899         d.addBoth(self.shouldRedirect, None, statuscode='303')
1900         def _check_target(target):
1901             target = urllib.unquote(target)
1902             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1903         d.addCallback(_check_target)
1904         return d
1905
1906     def test_POST_noparent_bad(self):
1907         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1908                                   "/uri accepts only PUT, PUT?t=mkdir, "
1909                                   "POST?t=upload, and POST?t=mkdir",
1910                                   self.POST, "/uri?t=bogus")
1911         return d
1912
1913     def test_welcome_page_mkdir_button(self):
1914         # Fetch the welcome page.
1915         d = self.GET("/")
1916         def _after_get_welcome_page(res):
1917             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1918             mo = MKDIR_BUTTON_RE.search(res)
1919             formaction = mo.group(1)
1920             formt = mo.group(2)
1921             formaname = mo.group(3)
1922             formavalue = mo.group(4)
1923             return (formaction, formt, formaname, formavalue)
1924         d.addCallback(_after_get_welcome_page)
1925         def _after_parse_form(res):
1926             (formaction, formt, formaname, formavalue) = res
1927             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1928         d.addCallback(_after_parse_form)
1929         d.addBoth(self.shouldRedirect, None, statuscode='303')
1930         return d
1931
1932     def test_POST_mkdir_replace(self): # return value?
1933         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1934         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1935         d.addCallback(self.failUnlessNodeKeysAre, [])
1936         return d
1937
1938     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1939         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1940         d.addBoth(self.shouldFail, error.Error,
1941                   "POST_mkdir_no_replace_queryarg",
1942                   "409 Conflict",
1943                   "There was already a child by that name, and you asked me "
1944                   "to not replace it")
1945         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1946         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1947         return d
1948
1949     def test_POST_mkdir_no_replace_field(self): # return value?
1950         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1951                       replace="false")
1952         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1953                   "409 Conflict",
1954                   "There was already a child by that name, and you asked me "
1955                   "to not replace it")
1956         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1957         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1958         return d
1959
1960     def test_POST_mkdir_whendone_field(self):
1961         d = self.POST(self.public_url + "/foo",
1962                       t="mkdir", name="newdir", when_done="/THERE")
1963         d.addBoth(self.shouldRedirect, "/THERE")
1964         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1965         d.addCallback(self.failUnlessNodeKeysAre, [])
1966         return d
1967
1968     def test_POST_mkdir_whendone_queryarg(self):
1969         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1970                       t="mkdir", name="newdir")
1971         d.addBoth(self.shouldRedirect, "/THERE")
1972         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1973         d.addCallback(self.failUnlessNodeKeysAre, [])
1974         return d
1975
1976     def test_POST_bad_t(self):
1977         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1978                              "POST to a directory with bad t=BOGUS",
1979                              self.POST, self.public_url + "/foo", t="BOGUS")
1980         return d
1981
1982     def test_POST_set_children(self):
1983         contents9, n9, newuri9 = self.makefile(9)
1984         contents10, n10, newuri10 = self.makefile(10)
1985         contents11, n11, newuri11 = self.makefile(11)
1986
1987         reqbody = """{
1988                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1989                                                 "size": 0,
1990                                                 "metadata": {
1991                                                   "ctime": 1002777696.7564139,
1992                                                   "mtime": 1002777696.7564139
1993                                                  }
1994                                                } ],
1995                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1996                                                 "size": 1,
1997                                                 "metadata": {
1998                                                   "ctime": 1002777696.7564139,
1999                                                   "mtime": 1002777696.7564139
2000                                                  }
2001                                                } ],
2002                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2003                                                 "size": 2,
2004                                                 "metadata": {
2005                                                   "ctime": 1002777696.7564139,
2006                                                   "mtime": 1002777696.7564139
2007                                                  }
2008                                                } ]
2009                     }""" % (newuri9, newuri10, newuri11)
2010
2011         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2012
2013         d = client.getPage(url, method="POST", postdata=reqbody)
2014         def _then(res):
2015             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2016             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2017             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2018
2019         d.addCallback(_then)
2020         d.addErrback(self.dump_error)
2021         return d
2022
2023     def test_POST_put_uri(self):
2024         contents, n, newuri = self.makefile(8)
2025         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2026         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2027         d.addCallback(lambda res:
2028                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2029                                                       contents))
2030         return d
2031
2032     def test_POST_put_uri_replace(self):
2033         contents, n, newuri = self.makefile(8)
2034         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2035         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2036         d.addCallback(lambda res:
2037                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2038                                                       contents))
2039         return d
2040
2041     def test_POST_put_uri_no_replace_queryarg(self):
2042         contents, n, newuri = self.makefile(8)
2043         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2044                       name="bar.txt", uri=newuri)
2045         d.addBoth(self.shouldFail, error.Error,
2046                   "POST_put_uri_no_replace_queryarg",
2047                   "409 Conflict",
2048                   "There was already a child by that name, and you asked me "
2049                   "to not replace it")
2050         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2051         d.addCallback(self.failUnlessIsBarDotTxt)
2052         return d
2053
2054     def test_POST_put_uri_no_replace_field(self):
2055         contents, n, newuri = self.makefile(8)
2056         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2057                       name="bar.txt", uri=newuri)
2058         d.addBoth(self.shouldFail, error.Error,
2059                   "POST_put_uri_no_replace_field",
2060                   "409 Conflict",
2061                   "There was already a child by that name, and you asked me "
2062                   "to not replace it")
2063         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2064         d.addCallback(self.failUnlessIsBarDotTxt)
2065         return d
2066
2067     def test_POST_delete(self):
2068         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2069         d.addCallback(lambda res: self._foo_node.list())
2070         def _check(children):
2071             self.failIf(u"bar.txt" in children)
2072         d.addCallback(_check)
2073         return d
2074
2075     def test_POST_rename_file(self):
2076         d = self.POST(self.public_url + "/foo", t="rename",
2077                       from_name="bar.txt", to_name='wibble.txt')
2078         d.addCallback(lambda res:
2079                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2080         d.addCallback(lambda res:
2081                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2082         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2083         d.addCallback(self.failUnlessIsBarDotTxt)
2084         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2085         d.addCallback(self.failUnlessIsBarJSON)
2086         return d
2087
2088     def test_POST_rename_file_redundant(self):
2089         d = self.POST(self.public_url + "/foo", t="rename",
2090                       from_name="bar.txt", to_name='bar.txt')
2091         d.addCallback(lambda res:
2092                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2093         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2094         d.addCallback(self.failUnlessIsBarDotTxt)
2095         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2096         d.addCallback(self.failUnlessIsBarJSON)
2097         return d
2098
2099     def test_POST_rename_file_replace(self):
2100         # rename a file and replace a directory with it
2101         d = self.POST(self.public_url + "/foo", t="rename",
2102                       from_name="bar.txt", to_name='empty')
2103         d.addCallback(lambda res:
2104                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2105         d.addCallback(lambda res:
2106                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2107         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2108         d.addCallback(self.failUnlessIsBarDotTxt)
2109         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2110         d.addCallback(self.failUnlessIsBarJSON)
2111         return d
2112
2113     def test_POST_rename_file_no_replace_queryarg(self):
2114         # rename a file and replace a directory with it
2115         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2116                       from_name="bar.txt", to_name='empty')
2117         d.addBoth(self.shouldFail, error.Error,
2118                   "POST_rename_file_no_replace_queryarg",
2119                   "409 Conflict",
2120                   "There was already a child by that name, and you asked me "
2121                   "to not replace it")
2122         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2123         d.addCallback(self.failUnlessIsEmptyJSON)
2124         return d
2125
2126     def test_POST_rename_file_no_replace_field(self):
2127         # rename a file and replace a directory with it
2128         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2129                       from_name="bar.txt", to_name='empty')
2130         d.addBoth(self.shouldFail, error.Error,
2131                   "POST_rename_file_no_replace_field",
2132                   "409 Conflict",
2133                   "There was already a child by that name, and you asked me "
2134                   "to not replace it")
2135         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2136         d.addCallback(self.failUnlessIsEmptyJSON)
2137         return d
2138
2139     def failUnlessIsEmptyJSON(self, res):
2140         data = simplejson.loads(res)
2141         self.failUnlessEqual(data[0], "dirnode", data)
2142         self.failUnlessEqual(len(data[1]["children"]), 0)
2143
2144     def test_POST_rename_file_slash_fail(self):
2145         d = self.POST(self.public_url + "/foo", t="rename",
2146                       from_name="bar.txt", to_name='kirk/spock.txt')
2147         d.addBoth(self.shouldFail, error.Error,
2148                   "test_POST_rename_file_slash_fail",
2149                   "400 Bad Request",
2150                   "to_name= may not contain a slash",
2151                   )
2152         d.addCallback(lambda res:
2153                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2154         return d
2155
2156     def test_POST_rename_dir(self):
2157         d = self.POST(self.public_url, t="rename",
2158                       from_name="foo", to_name='plunk')
2159         d.addCallback(lambda res:
2160                       self.failIfNodeHasChild(self.public_root, u"foo"))
2161         d.addCallback(lambda res:
2162                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2163         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2164         d.addCallback(self.failUnlessIsFooJSON)
2165         return d
2166
2167     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2168         """ If target is not None then the redirection has to go to target.  If
2169         statuscode is not None then the redirection has to be accomplished with
2170         that HTTP status code."""
2171         if not isinstance(res, failure.Failure):
2172             to_where = (target is None) and "somewhere" or ("to " + target)
2173             self.fail("%s: we were expecting to get redirected %s, not get an"
2174                       " actual page: %s" % (which, to_where, res))
2175         res.trap(error.PageRedirect)
2176         if statuscode is not None:
2177             self.failUnlessEqual(res.value.status, statuscode,
2178                                  "%s: not a redirect" % which)
2179         if target is not None:
2180             # the PageRedirect does not seem to capture the uri= query arg
2181             # properly, so we can't check for it.
2182             realtarget = self.webish_url + target
2183             self.failUnlessEqual(res.value.location, realtarget,
2184                                  "%s: wrong target" % which)
2185         return res.value.location
2186
2187     def test_GET_URI_form(self):
2188         base = "/uri?uri=%s" % self._bar_txt_uri
2189         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2190         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2191         d = self.GET(base)
2192         d.addBoth(self.shouldRedirect, targetbase)
2193         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2194         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2195         d.addCallback(lambda res: self.GET(base+"&t=json"))
2196         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2197         d.addCallback(self.log, "about to get file by uri")
2198         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2199         d.addCallback(self.failUnlessIsBarDotTxt)
2200         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2201         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2202                                            followRedirect=True))
2203         d.addCallback(self.failUnlessIsFooJSON)
2204         d.addCallback(self.log, "got dir by uri")
2205
2206         return d
2207
2208     def test_GET_URI_form_bad(self):
2209         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2210                              "400 Bad Request", "GET /uri requires uri=",
2211                              self.GET, "/uri")
2212         return d
2213
2214     def test_GET_rename_form(self):
2215         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2216                      followRedirect=True)
2217         def _check(res):
2218             self.failUnless('name="when_done" value="."' in res, res)
2219             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2220         d.addCallback(_check)
2221         return d
2222
2223     def log(self, res, msg):
2224         #print "MSG: %s  RES: %s" % (msg, res)
2225         log.msg(msg)
2226         return res
2227
2228     def test_GET_URI_URL(self):
2229         base = "/uri/%s" % self._bar_txt_uri
2230         d = self.GET(base)
2231         d.addCallback(self.failUnlessIsBarDotTxt)
2232         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2233         d.addCallback(self.failUnlessIsBarDotTxt)
2234         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2235         d.addCallback(self.failUnlessIsBarDotTxt)
2236         return d
2237
2238     def test_GET_URI_URL_dir(self):
2239         base = "/uri/%s?t=json" % self._foo_uri
2240         d = self.GET(base)
2241         d.addCallback(self.failUnlessIsFooJSON)
2242         return d
2243
2244     def test_GET_URI_URL_missing(self):
2245         base = "/uri/%s" % self._bad_file_uri
2246         d = self.GET(base)
2247         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2248                   http.GONE, response_substring="NotEnoughSharesError")
2249         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2250         # here? we must arrange for a download to fail after target.open()
2251         # has been called, and then inspect the response to see that it is
2252         # shorter than we expected.
2253         return d
2254
2255     def test_PUT_DIRURL_uri(self):
2256         d = self.s.create_empty_dirnode()
2257         def _made_dir(dn):
2258             new_uri = dn.get_uri()
2259             # replace /foo with a new (empty) directory
2260             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2261             d.addCallback(lambda res:
2262                           self.failUnlessEqual(res.strip(), new_uri))
2263             d.addCallback(lambda res:
2264                           self.failUnlessChildURIIs(self.public_root,
2265                                                     u"foo",
2266                                                     new_uri))
2267             return d
2268         d.addCallback(_made_dir)
2269         return d
2270
2271     def test_PUT_DIRURL_uri_noreplace(self):
2272         d = self.s.create_empty_dirnode()
2273         def _made_dir(dn):
2274             new_uri = dn.get_uri()
2275             # replace /foo with a new (empty) directory, but ask that
2276             # replace=false, so it should fail
2277             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2278                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2279                                  self.PUT,
2280                                  self.public_url + "/foo?t=uri&replace=false",
2281                                  new_uri)
2282             d.addCallback(lambda res:
2283                           self.failUnlessChildURIIs(self.public_root,
2284                                                     u"foo",
2285                                                     self._foo_uri))
2286             return d
2287         d.addCallback(_made_dir)
2288         return d
2289
2290     def test_PUT_DIRURL_bad_t(self):
2291         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2292                                  "400 Bad Request", "PUT to a directory",
2293                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2294         d.addCallback(lambda res:
2295                       self.failUnlessChildURIIs(self.public_root,
2296                                                 u"foo",
2297                                                 self._foo_uri))
2298         return d
2299
2300     def test_PUT_NEWFILEURL_uri(self):
2301         contents, n, new_uri = self.makefile(8)
2302         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2303         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2304         d.addCallback(lambda res:
2305                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2306                                                       contents))
2307         return d
2308
2309     def test_PUT_NEWFILEURL_uri_replace(self):
2310         contents, n, new_uri = self.makefile(8)
2311         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2312         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2313         d.addCallback(lambda res:
2314                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2315                                                       contents))
2316         return d
2317
2318     def test_PUT_NEWFILEURL_uri_no_replace(self):
2319         contents, n, new_uri = self.makefile(8)
2320         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2321         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2322                   "409 Conflict",
2323                   "There was already a child by that name, and you asked me "
2324                   "to not replace it")
2325         return d
2326
2327     def test_PUT_NEWFILE_URI(self):
2328         file_contents = "New file contents here\n"
2329         d = self.PUT("/uri", file_contents)
2330         def _check(uri):
2331             assert isinstance(uri, str), uri
2332             self.failUnless(uri in FakeCHKFileNode.all_contents)
2333             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2334                                  file_contents)
2335             return self.GET("/uri/%s" % uri)
2336         d.addCallback(_check)
2337         def _check2(res):
2338             self.failUnlessEqual(res, file_contents)
2339         d.addCallback(_check2)
2340         return d
2341
2342     def test_PUT_NEWFILE_URI_only_PUT(self):
2343         d = self.PUT("/uri?t=bogus", "")
2344         d.addBoth(self.shouldFail, error.Error,
2345                   "PUT_NEWFILE_URI_only_PUT",
2346                   "400 Bad Request",
2347                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2348         return d
2349
2350     def test_PUT_NEWFILE_URI_mutable(self):
2351         file_contents = "New file contents here\n"
2352         d = self.PUT("/uri?mutable=true", file_contents)
2353         def _check_mutable(uri):
2354             uri = uri.strip()
2355             u = IURI(uri)
2356             self.failUnless(IMutableFileURI.providedBy(u))
2357             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2358             n = self.s.create_node_from_uri(uri)
2359             return n.download_best_version()
2360         d.addCallback(_check_mutable)
2361         def _check2_mutable(data):
2362             self.failUnlessEqual(data, file_contents)
2363         d.addCallback(_check2_mutable)
2364         return d
2365
2366         def _check(uri):
2367             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2368             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2369                                  file_contents)
2370             return self.GET("/uri/%s" % uri)
2371         d.addCallback(_check)
2372         def _check2(res):
2373             self.failUnlessEqual(res, file_contents)
2374         d.addCallback(_check2)
2375         return d
2376
2377     def test_PUT_mkdir(self):
2378         d = self.PUT("/uri?t=mkdir", "")
2379         def _check(uri):
2380             n = self.s.create_node_from_uri(uri.strip())
2381             d2 = self.failUnlessNodeKeysAre(n, [])
2382             d2.addCallback(lambda res:
2383                            self.GET("/uri/%s?t=json" % uri))
2384             return d2
2385         d.addCallback(_check)
2386         d.addCallback(self.failUnlessIsEmptyJSON)
2387         return d
2388
2389     def test_POST_check(self):
2390         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2391         def _done(res):
2392             # this returns a string form of the results, which are probably
2393             # None since we're using fake filenodes.
2394             # TODO: verify that the check actually happened, by changing
2395             # FakeCHKFileNode to count how many times .check() has been
2396             # called.
2397             pass
2398         d.addCallback(_done)
2399         return d
2400
2401     def test_bad_method(self):
2402         url = self.webish_url + self.public_url + "/foo/bar.txt"
2403         d = self.shouldHTTPError2("test_bad_method",
2404                                   501, "Not Implemented",
2405                                   "I don't know how to treat a BOGUS request.",
2406                                   client.getPage, url, method="BOGUS")
2407         return d
2408
2409     def test_short_url(self):
2410         url = self.webish_url + "/uri"
2411         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2412                                   "I don't know how to treat a DELETE request.",
2413                                   client.getPage, url, method="DELETE")
2414         return d
2415
2416     def test_ophandle_bad(self):
2417         url = self.webish_url + "/operations/bogus?t=status"
2418         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2419                                   "unknown/expired handle 'bogus'",
2420                                   client.getPage, url)
2421         return d
2422
2423     def test_ophandle_cancel(self):
2424         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2425                       followRedirect=True)
2426         d.addCallback(lambda ignored:
2427                       self.GET("/operations/128?t=status&output=JSON"))
2428         def _check1(res):
2429             data = simplejson.loads(res)
2430             self.failUnless("finished" in data, res)
2431             monitor = self.ws.root.child_operations.handles["128"][0]
2432             d = self.POST("/operations/128?t=cancel&output=JSON")
2433             def _check2(res):
2434                 data = simplejson.loads(res)
2435                 self.failUnless("finished" in data, res)
2436                 # t=cancel causes the handle to be forgotten
2437                 self.failUnless(monitor.is_cancelled())
2438             d.addCallback(_check2)
2439             return d
2440         d.addCallback(_check1)
2441         d.addCallback(lambda ignored:
2442                       self.shouldHTTPError2("test_ophandle_cancel",
2443                                             404, "404 Not Found",
2444                                             "unknown/expired handle '128'",
2445                                             self.GET,
2446                                             "/operations/128?t=status&output=JSON"))
2447         return d
2448
2449     def test_ophandle_retainfor(self):
2450         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2451                       followRedirect=True)
2452         d.addCallback(lambda ignored:
2453                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2454         def _check1(res):
2455             data = simplejson.loads(res)
2456             self.failUnless("finished" in data, res)
2457         d.addCallback(_check1)
2458         # the retain-for=0 will cause the handle to be expired very soon
2459         d.addCallback(self.stall, 2.0)
2460         d.addCallback(lambda ignored:
2461                       self.shouldHTTPError2("test_ophandle_retainfor",
2462                                             404, "404 Not Found",
2463                                             "unknown/expired handle '129'",
2464                                             self.GET,
2465                                             "/operations/129?t=status&output=JSON"))
2466         return d
2467
2468     def test_ophandle_release_after_complete(self):
2469         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2470                       followRedirect=True)
2471         d.addCallback(self.wait_for_operation, "130")
2472         d.addCallback(lambda ignored:
2473                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2474         # the release-after-complete=true will cause the handle to be expired
2475         d.addCallback(lambda ignored:
2476                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2477                                             404, "404 Not Found",
2478                                             "unknown/expired handle '130'",
2479                                             self.GET,
2480                                             "/operations/130?t=status&output=JSON"))
2481         return d
2482
2483     def test_incident(self):
2484         d = self.POST("/report_incident", details="eek")
2485         def _done(res):
2486             self.failUnless("Thank you for your report!" in res, res)
2487         d.addCallback(_done)
2488         return d
2489
2490     def test_static(self):
2491         webdir = os.path.join(self.staticdir, "subdir")
2492         fileutil.make_dirs(webdir)
2493         f = open(os.path.join(webdir, "hello.txt"), "wb")
2494         f.write("hello")
2495         f.close()
2496
2497         d = self.GET("/static/subdir/hello.txt")
2498         def _check(res):
2499             self.failUnlessEqual(res, "hello")
2500         d.addCallback(_check)
2501         return d
2502
2503
2504 class Util(unittest.TestCase):
2505     def test_abbreviate_time(self):
2506         self.failUnlessEqual(common.abbreviate_time(None), "")
2507         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2508         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2509         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2510         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2511
2512     def test_abbreviate_rate(self):
2513         self.failUnlessEqual(common.abbreviate_rate(None), "")
2514         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2515         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2516         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2517
2518     def test_abbreviate_size(self):
2519         self.failUnlessEqual(common.abbreviate_size(None), "")
2520         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2521         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2522         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2523         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2524
2525
2526 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase):
2527
2528     def GET(self, urlpath, followRedirect=False, return_response=False,
2529             method="GET", clientnum=0, **kwargs):
2530         # if return_response=True, this fires with (data, statuscode,
2531         # respheaders) instead of just data.
2532         assert not isinstance(urlpath, unicode)
2533         url = self.client_baseurls[clientnum] + urlpath
2534         factory = HTTPClientGETFactory(url, method=method,
2535                                        followRedirect=followRedirect, **kwargs)
2536         reactor.connectTCP("localhost", self.client_webports[clientnum],factory)
2537         d = factory.deferred
2538         def _got_data(data):
2539             return (data, factory.status, factory.response_headers)
2540         if return_response:
2541             d.addCallback(_got_data)
2542         return factory.deferred
2543
2544     def CHECK(self, ign, which, args, clientnum=0):
2545         fileurl = self.fileurls[which]
2546         url = fileurl + "?" + args
2547         return self.GET(url, method="POST", clientnum=clientnum)
2548
2549     def test_filecheck(self):
2550         self.basedir = "web/Grid/filecheck"
2551         self.set_up_grid()
2552         c0 = self.g.clients[0]
2553         self.uris = {}
2554         DATA = "data" * 100
2555         d = c0.upload(upload.Data(DATA, convergence=""))
2556         def _stash_uri(ur, which):
2557             self.uris[which] = ur.uri
2558         d.addCallback(_stash_uri, "good")
2559         d.addCallback(lambda ign:
2560                       c0.upload(upload.Data(DATA+"1", convergence="")))
2561         d.addCallback(_stash_uri, "sick")
2562         d.addCallback(lambda ign:
2563                       c0.upload(upload.Data(DATA+"2", convergence="")))
2564         d.addCallback(_stash_uri, "dead")
2565         def _stash_mutable_uri(n, which):
2566             self.uris[which] = n.get_uri()
2567             assert isinstance(self.uris[which], str)
2568         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2569         d.addCallback(_stash_mutable_uri, "corrupt")
2570         d.addCallback(lambda ign:
2571                       c0.upload(upload.Data("literal", convergence="")))
2572         d.addCallback(_stash_uri, "small")
2573
2574         def _compute_fileurls(ignored):
2575             self.fileurls = {}
2576             for which in self.uris:
2577                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2578         d.addCallback(_compute_fileurls)
2579
2580         def _clobber_shares(ignored):
2581             good_shares = self.find_shares(self.uris["good"])
2582             self.failUnlessEqual(len(good_shares), 10)
2583             sick_shares = self.find_shares(self.uris["sick"])
2584             os.unlink(sick_shares[0][2])
2585             dead_shares = self.find_shares(self.uris["dead"])
2586             for i in range(1, 10):
2587                 os.unlink(dead_shares[i][2])
2588             c_shares = self.find_shares(self.uris["corrupt"])
2589             cso = CorruptShareOptions()
2590             cso.stdout = StringIO()
2591             cso.parseOptions([c_shares[0][2]])
2592             corrupt_share(cso)
2593         d.addCallback(_clobber_shares)
2594
2595         d.addCallback(self.CHECK, "good", "t=check")
2596         def _got_html_good(res):
2597             self.failUnless("Healthy" in res, res)
2598             self.failIf("Not Healthy" in res, res)
2599         d.addCallback(_got_html_good)
2600         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2601         def _got_html_good_return_to(res):
2602             self.failUnless("Healthy" in res, res)
2603             self.failIf("Not Healthy" in res, res)
2604             self.failUnless('<a href="somewhere">Return to parent directory'
2605                             in res, res)
2606         d.addCallback(_got_html_good_return_to)
2607         d.addCallback(self.CHECK, "good", "t=check&output=json")
2608         def _got_json_good(res):
2609             r = simplejson.loads(res)
2610             self.failUnlessEqual(r["summary"], "Healthy")
2611             self.failUnless(r["results"]["healthy"])
2612             self.failIf(r["results"]["needs-rebalancing"])
2613             self.failUnless(r["results"]["recoverable"])
2614         d.addCallback(_got_json_good)
2615
2616         d.addCallback(self.CHECK, "small", "t=check")
2617         def _got_html_small(res):
2618             self.failUnless("Literal files are always healthy" in res, res)
2619             self.failIf("Not Healthy" in res, res)
2620         d.addCallback(_got_html_small)
2621         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2622         def _got_html_small_return_to(res):
2623             self.failUnless("Literal files are always healthy" in res, res)
2624             self.failIf("Not Healthy" in res, res)
2625             self.failUnless('<a href="somewhere">Return to parent directory'
2626                             in res, res)
2627         d.addCallback(_got_html_small_return_to)
2628         d.addCallback(self.CHECK, "small", "t=check&output=json")
2629         def _got_json_small(res):
2630             r = simplejson.loads(res)
2631             self.failUnlessEqual(r["storage-index"], "")
2632             self.failUnless(r["results"]["healthy"])
2633         d.addCallback(_got_json_small)
2634
2635         d.addCallback(self.CHECK, "sick", "t=check")
2636         def _got_html_sick(res):
2637             self.failUnless("Not Healthy" in res, res)
2638         d.addCallback(_got_html_sick)
2639         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2640         def _got_json_sick(res):
2641             r = simplejson.loads(res)
2642             self.failUnlessEqual(r["summary"],
2643                                  "Not Healthy: 9 shares (enc 3-of-10)")
2644             self.failIf(r["results"]["healthy"])
2645             self.failIf(r["results"]["needs-rebalancing"])
2646             self.failUnless(r["results"]["recoverable"])
2647         d.addCallback(_got_json_sick)
2648
2649         d.addCallback(self.CHECK, "dead", "t=check")
2650         def _got_html_dead(res):
2651             self.failUnless("Not Healthy" in res, res)
2652         d.addCallback(_got_html_dead)
2653         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2654         def _got_json_dead(res):
2655             r = simplejson.loads(res)
2656             self.failUnlessEqual(r["summary"],
2657                                  "Not Healthy: 1 shares (enc 3-of-10)")
2658             self.failIf(r["results"]["healthy"])
2659             self.failIf(r["results"]["needs-rebalancing"])
2660             self.failIf(r["results"]["recoverable"])
2661         d.addCallback(_got_json_dead)
2662
2663         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2664         def _got_html_corrupt(res):
2665             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2666         d.addCallback(_got_html_corrupt)
2667         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2668         def _got_json_corrupt(res):
2669             r = simplejson.loads(res)
2670             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2671                             r["summary"])
2672             self.failIf(r["results"]["healthy"])
2673             self.failUnless(r["results"]["recoverable"])
2674             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2675             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2676         d.addCallback(_got_json_corrupt)
2677
2678         d.addErrback(self.explain_web_error)
2679         return d
2680
2681     def test_repair_html(self):
2682         self.basedir = "web/Grid/repair_html"
2683         self.set_up_grid()
2684         c0 = self.g.clients[0]
2685         self.uris = {}
2686         DATA = "data" * 100
2687         d = c0.upload(upload.Data(DATA, convergence=""))
2688         def _stash_uri(ur, which):
2689             self.uris[which] = ur.uri
2690         d.addCallback(_stash_uri, "good")
2691         d.addCallback(lambda ign:
2692                       c0.upload(upload.Data(DATA+"1", convergence="")))
2693         d.addCallback(_stash_uri, "sick")
2694         d.addCallback(lambda ign:
2695                       c0.upload(upload.Data(DATA+"2", convergence="")))
2696         d.addCallback(_stash_uri, "dead")
2697         def _stash_mutable_uri(n, which):
2698             self.uris[which] = n.get_uri()
2699             assert isinstance(self.uris[which], str)
2700         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2701         d.addCallback(_stash_mutable_uri, "corrupt")
2702
2703         def _compute_fileurls(ignored):
2704             self.fileurls = {}
2705             for which in self.uris:
2706                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2707         d.addCallback(_compute_fileurls)
2708
2709         def _clobber_shares(ignored):
2710             good_shares = self.find_shares(self.uris["good"])
2711             self.failUnlessEqual(len(good_shares), 10)
2712             sick_shares = self.find_shares(self.uris["sick"])
2713             os.unlink(sick_shares[0][2])
2714             dead_shares = self.find_shares(self.uris["dead"])
2715             for i in range(1, 10):
2716                 os.unlink(dead_shares[i][2])
2717             c_shares = self.find_shares(self.uris["corrupt"])
2718             cso = CorruptShareOptions()
2719             cso.stdout = StringIO()
2720             cso.parseOptions([c_shares[0][2]])
2721             corrupt_share(cso)
2722         d.addCallback(_clobber_shares)
2723
2724         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2725         def _got_html_good(res):
2726             self.failUnless("Healthy" in res, res)
2727             self.failIf("Not Healthy" in res, res)
2728             self.failUnless("No repair necessary" in res, res)
2729         d.addCallback(_got_html_good)
2730
2731         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2732         def _got_html_sick(res):
2733             self.failUnless("Healthy : healthy" in res, res)
2734             self.failIf("Not Healthy" in res, res)
2735             self.failUnless("Repair successful" in res, res)
2736         d.addCallback(_got_html_sick)
2737
2738         # repair of a dead file will fail, of course, but it isn't yet
2739         # clear how this should be reported. Right now it shows up as
2740         # a "410 Gone".
2741         #
2742         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2743         #def _got_html_dead(res):
2744         #    print res
2745         #    self.failUnless("Healthy : healthy" in res, res)
2746         #    self.failIf("Not Healthy" in res, res)
2747         #    self.failUnless("No repair necessary" in res, res)
2748         #d.addCallback(_got_html_dead)
2749
2750         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2751         def _got_html_corrupt(res):
2752             self.failUnless("Healthy : Healthy" in res, res)
2753             self.failIf("Not Healthy" in res, res)
2754             self.failUnless("Repair successful" in res, res)
2755         d.addCallback(_got_html_corrupt)
2756
2757         d.addErrback(self.explain_web_error)
2758         return d
2759
2760     def test_repair_json(self):
2761         self.basedir = "web/Grid/repair_json"
2762         self.set_up_grid()
2763         c0 = self.g.clients[0]
2764         self.uris = {}
2765         DATA = "data" * 100
2766         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2767         def _stash_uri(ur, which):
2768             self.uris[which] = ur.uri
2769         d.addCallback(_stash_uri, "sick")
2770
2771         def _compute_fileurls(ignored):
2772             self.fileurls = {}
2773             for which in self.uris:
2774                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2775         d.addCallback(_compute_fileurls)
2776
2777         def _clobber_shares(ignored):
2778             sick_shares = self.find_shares(self.uris["sick"])
2779             os.unlink(sick_shares[0][2])
2780         d.addCallback(_clobber_shares)
2781
2782         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2783         def _got_json_sick(res):
2784             r = simplejson.loads(res)
2785             self.failUnlessEqual(r["repair-attempted"], True)
2786             self.failUnlessEqual(r["repair-successful"], True)
2787             self.failUnlessEqual(r["pre-repair-results"]["summary"],
2788                                  "Not Healthy: 9 shares (enc 3-of-10)")
2789             self.failIf(r["pre-repair-results"]["results"]["healthy"])
2790             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2791             self.failUnless(r["post-repair-results"]["results"]["healthy"])
2792         d.addCallback(_got_json_sick)
2793
2794         d.addErrback(self.explain_web_error)
2795         return d
2796
2797     def test_deep_check(self):
2798         self.basedir = "web/Grid/deep_check"
2799         self.set_up_grid()
2800         c0 = self.g.clients[0]
2801         self.uris = {}
2802         self.fileurls = {}
2803         DATA = "data" * 100
2804         d = c0.create_empty_dirnode()
2805         def _stash_root_and_create_file(n):
2806             self.rootnode = n
2807             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2808             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2809         d.addCallback(_stash_root_and_create_file)
2810         def _stash_uri(fn, which):
2811             self.uris[which] = fn.get_uri()
2812         d.addCallback(_stash_uri, "good")
2813         d.addCallback(lambda ign:
2814                       self.rootnode.add_file(u"small",
2815                                              upload.Data("literal",
2816                                                         convergence="")))
2817         d.addCallback(_stash_uri, "small")
2818
2819         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2820         def _done(res):
2821             units = [simplejson.loads(line)
2822                      for line in res.splitlines()
2823                      if line]
2824             self.failUnlessEqual(len(units), 3+1)
2825             # should be parent-first
2826             u0 = units[0]
2827             self.failUnlessEqual(u0["path"], [])
2828             self.failUnlessEqual(u0["type"], "directory")
2829             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2830             u0cr = u0["check-results"]
2831             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2832
2833             ugood = [u for u in units
2834                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2835             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2836             ugoodcr = ugood["check-results"]
2837             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2838
2839             stats = units[-1]
2840             self.failUnlessEqual(stats["type"], "stats")
2841             s = stats["stats"]
2842             self.failUnlessEqual(s["count-immutable-files"], 1)
2843             self.failUnlessEqual(s["count-literal-files"], 1)
2844             self.failUnlessEqual(s["count-directories"], 1)
2845         d.addCallback(_done)
2846
2847         d.addErrback(self.explain_web_error)
2848         return d
2849
2850     def test_deep_check_and_repair(self):
2851         self.basedir = "web/Grid/deep_check_and_repair"
2852         self.set_up_grid()
2853         c0 = self.g.clients[0]
2854         self.uris = {}
2855         self.fileurls = {}
2856         DATA = "data" * 100
2857         d = c0.create_empty_dirnode()
2858         def _stash_root_and_create_file(n):
2859             self.rootnode = n
2860             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2861             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2862         d.addCallback(_stash_root_and_create_file)
2863         def _stash_uri(fn, which):
2864             self.uris[which] = fn.get_uri()
2865         d.addCallback(_stash_uri, "good")
2866         d.addCallback(lambda ign:
2867                       self.rootnode.add_file(u"small",
2868                                              upload.Data("literal",
2869                                                         convergence="")))
2870         d.addCallback(_stash_uri, "small")
2871         d.addCallback(lambda ign:
2872                       self.rootnode.add_file(u"sick",
2873                                              upload.Data(DATA+"1",
2874                                                         convergence="")))
2875         d.addCallback(_stash_uri, "sick")
2876         #d.addCallback(lambda ign:
2877         #              self.rootnode.add_file(u"dead",
2878         #                                     upload.Data(DATA+"2",
2879         #                                                convergence="")))
2880         #d.addCallback(_stash_uri, "dead")
2881
2882         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2883         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2884         #d.addCallback(_stash_uri, "corrupt")
2885
2886         def _clobber_shares(ignored):
2887             good_shares = self.find_shares(self.uris["good"])
2888             self.failUnlessEqual(len(good_shares), 10)
2889             sick_shares = self.find_shares(self.uris["sick"])
2890             os.unlink(sick_shares[0][2])
2891             #dead_shares = self.find_shares(self.uris["dead"])
2892             #for i in range(1, 10):
2893             #    os.unlink(dead_shares[i][2])
2894
2895             #c_shares = self.find_shares(self.uris["corrupt"])
2896             #cso = CorruptShareOptions()
2897             #cso.stdout = StringIO()
2898             #cso.parseOptions([c_shares[0][2]])
2899             #corrupt_share(cso)
2900         d.addCallback(_clobber_shares)
2901
2902         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2903         def _done(res):
2904             units = [simplejson.loads(line)
2905                      for line in res.splitlines()
2906                      if line]
2907             self.failUnlessEqual(len(units), 4+1)
2908             # should be parent-first
2909             u0 = units[0]
2910             self.failUnlessEqual(u0["path"], [])
2911             self.failUnlessEqual(u0["type"], "directory")
2912             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2913             u0crr = u0["check-and-repair-results"]
2914             self.failUnlessEqual(u0crr["repair-attempted"], False)
2915             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2916
2917             ugood = [u for u in units
2918                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2919             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2920             ugoodcrr = ugood["check-and-repair-results"]
2921             self.failUnlessEqual(u0crr["repair-attempted"], False)
2922             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2923
2924             usick = [u for u in units
2925                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
2926             self.failUnlessEqual(usick["cap"], self.uris["sick"])
2927             usickcrr = usick["check-and-repair-results"]
2928             self.failUnlessEqual(usickcrr["repair-attempted"], True)
2929             self.failUnlessEqual(usickcrr["repair-successful"], True)
2930             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2931             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2932
2933             stats = units[-1]
2934             self.failUnlessEqual(stats["type"], "stats")
2935             s = stats["stats"]
2936             self.failUnlessEqual(s["count-immutable-files"], 2)
2937             self.failUnlessEqual(s["count-literal-files"], 1)
2938             self.failUnlessEqual(s["count-directories"], 1)
2939         d.addCallback(_done)
2940
2941         d.addErrback(self.explain_web_error)
2942         return d
2943
2944     def _count_leases(self, ignored, which):
2945         u = self.uris[which]
2946         shares = self.find_shares(u)
2947         lease_counts = []
2948         for shnum, serverid, fn in shares:
2949             if u.startswith("URI:SSK") or u.startswith("URI:DIR2"):
2950                 sf = storage.MutableShareFile(fn)
2951                 num_leases = len(sf.debug_get_leases())
2952             elif u.startswith("URI:CHK"):
2953                 sf = storage.ShareFile(fn)
2954                 num_leases = len(list(sf.iter_leases()))
2955             else:
2956                 raise RuntimeError("can't get leases on %s" % u)
2957         lease_counts.append( (fn, num_leases) )
2958         return lease_counts
2959
2960     def _assert_leasecount(self, lease_counts, expected):
2961         for (fn, num_leases) in lease_counts:
2962             if num_leases != expected:
2963                 self.fail("expected %d leases, have %d, on %s" %
2964                           (expected, num_leases, fn))
2965
2966     def test_add_lease(self):
2967         self.basedir = "web/Grid/add_lease"
2968         self.set_up_grid(num_clients=2)
2969         c0 = self.g.clients[0]
2970         self.uris = {}
2971         DATA = "data" * 100
2972         d = c0.upload(upload.Data(DATA, convergence=""))
2973         def _stash_uri(ur, which):
2974             self.uris[which] = ur.uri
2975         d.addCallback(_stash_uri, "one")
2976         d.addCallback(lambda ign:
2977                       c0.upload(upload.Data(DATA+"1", convergence="")))
2978         d.addCallback(_stash_uri, "two")
2979         def _stash_mutable_uri(n, which):
2980             self.uris[which] = n.get_uri()
2981             assert isinstance(self.uris[which], str)
2982         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
2983         d.addCallback(_stash_mutable_uri, "mutable")
2984
2985         def _compute_fileurls(ignored):
2986             self.fileurls = {}
2987             for which in self.uris:
2988                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2989         d.addCallback(_compute_fileurls)
2990
2991         d.addCallback(self._count_leases, "one")
2992         d.addCallback(self._assert_leasecount, 1)
2993         d.addCallback(self._count_leases, "two")
2994         d.addCallback(self._assert_leasecount, 1)
2995         d.addCallback(self._count_leases, "mutable")
2996         d.addCallback(self._assert_leasecount, 1)
2997
2998         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
2999         def _got_html_good(res):
3000             self.failUnless("Healthy" in res, res)
3001             self.failIf("Not Healthy" in res, res)
3002         d.addCallback(_got_html_good)
3003
3004         d.addCallback(self._count_leases, "one")
3005         d.addCallback(self._assert_leasecount, 1)
3006         d.addCallback(self._count_leases, "two")
3007         d.addCallback(self._assert_leasecount, 1)
3008         d.addCallback(self._count_leases, "mutable")
3009         d.addCallback(self._assert_leasecount, 1)
3010
3011         # this CHECK uses the original client, which uses the same
3012         # lease-secrets, so it will just renew the original lease
3013         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3014         d.addCallback(_got_html_good)
3015
3016         d.addCallback(self._count_leases, "one")
3017         d.addCallback(self._assert_leasecount, 1)
3018         d.addCallback(self._count_leases, "two")
3019         d.addCallback(self._assert_leasecount, 1)
3020         d.addCallback(self._count_leases, "mutable")
3021         d.addCallback(self._assert_leasecount, 1)
3022
3023         # this CHECK uses an alternate client, which adds a second lease
3024         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3025         d.addCallback(_got_html_good)
3026
3027         d.addCallback(self._count_leases, "one")
3028         d.addCallback(self._assert_leasecount, 2)
3029         d.addCallback(self._count_leases, "two")
3030         d.addCallback(self._assert_leasecount, 1)
3031         d.addCallback(self._count_leases, "mutable")
3032         d.addCallback(self._assert_leasecount, 1)
3033
3034         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3035         d.addCallback(_got_html_good)
3036
3037         d.addCallback(self._count_leases, "one")
3038         d.addCallback(self._assert_leasecount, 2)
3039         d.addCallback(self._count_leases, "two")
3040         d.addCallback(self._assert_leasecount, 1)
3041         d.addCallback(self._count_leases, "mutable")
3042         d.addCallback(self._assert_leasecount, 1)
3043
3044         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3045                       clientnum=1)
3046         d.addCallback(_got_html_good)
3047
3048         d.addCallback(self._count_leases, "one")
3049         d.addCallback(self._assert_leasecount, 2)
3050         d.addCallback(self._count_leases, "two")
3051         d.addCallback(self._assert_leasecount, 1)
3052         d.addCallback(self._count_leases, "mutable")
3053         d.addCallback(self._assert_leasecount, 2)
3054
3055         d.addErrback(self.explain_web_error)
3056         return d
3057
3058     def test_deep_add_lease(self):
3059         self.basedir = "web/Grid/deep_add_lease"
3060         self.set_up_grid(num_clients=2)
3061         c0 = self.g.clients[0]
3062         self.uris = {}
3063         self.fileurls = {}
3064         DATA = "data" * 100
3065         d = c0.create_empty_dirnode()
3066         def _stash_root_and_create_file(n):
3067             self.rootnode = n
3068             self.uris["root"] = n.get_uri()
3069             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3070             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3071         d.addCallback(_stash_root_and_create_file)
3072         def _stash_uri(fn, which):
3073             self.uris[which] = fn.get_uri()
3074         d.addCallback(_stash_uri, "one")
3075         d.addCallback(lambda ign:
3076                       self.rootnode.add_file(u"small",
3077                                              upload.Data("literal",
3078                                                         convergence="")))
3079         d.addCallback(_stash_uri, "small")
3080
3081         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3082         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3083         d.addCallback(_stash_uri, "mutable")
3084
3085         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3086         def _done(res):
3087             units = [simplejson.loads(line)
3088                      for line in res.splitlines()
3089                      if line]
3090             # root, one, small, mutable,   stats
3091             self.failUnlessEqual(len(units), 4+1)
3092         d.addCallback(_done)
3093
3094         d.addCallback(self._count_leases, "root")
3095         d.addCallback(self._assert_leasecount, 1)
3096         d.addCallback(self._count_leases, "one")
3097         d.addCallback(self._assert_leasecount, 1)
3098         d.addCallback(self._count_leases, "mutable")
3099         d.addCallback(self._assert_leasecount, 1)
3100
3101         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3102         d.addCallback(_done)
3103
3104         d.addCallback(self._count_leases, "root")
3105         d.addCallback(self._assert_leasecount, 1)
3106         d.addCallback(self._count_leases, "one")
3107         d.addCallback(self._assert_leasecount, 1)
3108         d.addCallback(self._count_leases, "mutable")
3109         d.addCallback(self._assert_leasecount, 1)
3110
3111         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3112                       clientnum=1)
3113         d.addCallback(_done)
3114
3115         d.addCallback(self._count_leases, "root")
3116         d.addCallback(self._assert_leasecount, 2)
3117         d.addCallback(self._count_leases, "one")
3118         d.addCallback(self._assert_leasecount, 2)
3119         d.addCallback(self._count_leases, "mutable")
3120         d.addCallback(self._assert_leasecount, 2)
3121
3122         d.addErrback(self.explain_web_error)
3123         return d