src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from allmydata import interfaces, uri, webish
  10 from allmydata.storage.mutable import MutableShareFile
  11 from allmydata.storage.immutable import ShareFile
  12 from allmydata.immutable import upload, download
  13 from allmydata.web import status, common
  14 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  15 from allmydata.util import fileutil, base32
  16 from allmydata.util.assertutil import precondition
  17 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  18      FakeMutableFileNode, create_chk_filenode, WebErrorMixin
  19 from allmydata.interfaces import IURI, INewDirectoryURI, \
  20      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  21 from allmydata.mutable import servermap, publish, retrieve
  22 import common_util as testutil
  23 from allmydata.test.no_network import GridTestMixin
  24
  25 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  26 # create a webserver that works against them
  27
  28 class FakeIntroducerClient:
  29     def get_all_connectors(self):
  30         return {}
  31     def get_all_connections_for(self, service_name):
  32         return frozenset()
  33     def get_all_peerids(self):
  34         return frozenset()
  35
  36 class FakeStatsProvider:
  37     def get_stats(self):
  38         stats = {'stats': {}, 'counters': {}}
  39         return stats
  40
  41 class FakeClient(service.MultiService):
  42     nodeid = "fake_nodeid"
  43     nickname = "fake_nickname"
  44     basedir = "fake_basedir"
  45     def get_versions(self):
  46         return {'allmydata': "fake",
  47                 'foolscap': "fake",
  48                 'twisted': "fake",
  49                 'zfec': "fake",
  50                 }
  51     introducer_furl = "None"
  52     introducer_client = FakeIntroducerClient()
  53     _all_upload_status = [upload.UploadStatus()]
  54     _all_download_status = [download.DownloadStatus()]
  55     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  56     _all_publish_statuses = [publish.PublishStatus()]
  57     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  58     convergence = "some random string"
  59     stats_provider = FakeStatsProvider()
  60
  61     def connected_to_introducer(self):
  62         return False
  63
  64     def get_nickname_for_peerid(self, peerid):
  65         return u"John Doe"
  66
  67     def get_permuted_peers(self, service_name, key):
  68         return []
  69
  70     def create_node_from_uri(self, auri):
  71         precondition(isinstance(auri, str), auri)
  72         u = uri.from_string(auri)
  73         if (INewDirectoryURI.providedBy(u)
  74             or IReadonlyNewDirectoryURI.providedBy(u)):
  75             return FakeDirectoryNode(self).init_from_uri(u)
  76         if IFileURI.providedBy(u):
  77             return FakeCHKFileNode(u, self)
  78         assert IMutableFileURI.providedBy(u), u
  79         return FakeMutableFileNode(self).init_from_uri(u)
  80
  81     def create_empty_dirnode(self):
  82         n = FakeDirectoryNode(self)
  83         d = n.create()
  84         d.addCallback(lambda res: n)
  85         return d
  86
  87     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  88     def create_mutable_file(self, contents=""):
  89         n = FakeMutableFileNode(self)
  90         return n.create(contents)
  91
  92     def upload(self, uploadable):
  93         d = uploadable.get_size()
  94         d.addCallback(lambda size: uploadable.read(size))
  95         def _got_data(datav):
  96             data = "".join(datav)
  97             n = create_chk_filenode(self, data)
  98             results = upload.UploadResults()
  99             results.uri = n.get_uri()
 100             return results
 101         d.addCallback(_got_data)
 102         return d
 103
 104     def list_all_upload_statuses(self):
 105         return self._all_upload_status
 106     def list_all_download_statuses(self):
 107         return self._all_download_status
 108     def list_all_mapupdate_statuses(self):
 109         return self._all_mapupdate_statuses
 110     def list_all_publish_statuses(self):
 111         return self._all_publish_statuses
 112     def list_all_retrieve_statuses(self):
 113         return self._all_retrieve_statuses
 114     def list_all_helper_statuses(self):
 115         return []
 116
 117 class MyGetter(client.HTTPPageGetter):
 118     handleStatus_206 = lambda self: self.handleStatus_200()
 119
 120 class HTTPClientHEADFactory(client.HTTPClientFactory):
 121     protocol = MyGetter
 122
 123     def noPage(self, reason):
 124         # Twisted-2.5.0 and earlier had a bug, in which they would raise an
 125         # exception when the response to a HEAD request had no body (when in
 126         # fact they are defined to never have a body). This was fixed in
 127         # Twisted-8.0 . To work around this, we catch the
 128         # PartialDownloadError and make it disappear.
 129         if (reason.check(client.PartialDownloadError)
 130             and self.method.upper() == "HEAD"):
 131             self.page("")
 132             return
 133         return client.HTTPClientFactory.noPage(self, reason)
 134
 135 class HTTPClientGETFactory(client.HTTPClientFactory):
 136     protocol = MyGetter
 137
 138 class WebMixin(object):
 139     def setUp(self):
 140         self.s = FakeClient()
 141         self.s.startService()
 142         self.staticdir = self.mktemp()
 143         self.ws = s = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 144         s.setServiceParent(self.s)
 145         self.webish_port = port = s.listener._port.getHost().port
 146         self.webish_url = "http://localhost:%d" % port
 147
 148         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 149         d = defer.DeferredList(l)
 150         def _then(res):
 151             self.public_root = res[0][1]
 152             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 153             self.public_url = "/uri/" + self.public_root.get_uri()
 154             self.private_root = res[1][1]
 155
 156             foo = res[2][1]
 157             self._foo_node = foo
 158             self._foo_uri = foo.get_uri()
 159             self._foo_readonly_uri = foo.get_readonly_uri()
 160             self._foo_verifycap = foo.get_verify_cap().to_string()
 161             # NOTE: we ignore the deferred on all set_uri() calls, because we
 162             # know the fake nodes do these synchronously
 163             self.public_root.set_uri(u"foo", foo.get_uri())
 164
 165             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 166             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 167             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 168
 169             foo.set_uri(u"empty", res[3][1].get_uri())
 170             sub_uri = res[4][1].get_uri()
 171             self._sub_uri = sub_uri
 172             foo.set_uri(u"sub", sub_uri)
 173             sub = self.s.create_node_from_uri(sub_uri)
 174
 175             _ign, n, blocking_uri = self.makefile(1)
 176             foo.set_uri(u"blockingfile", blocking_uri)
 177
 178             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 179             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 180             # still think of it as an umlaut
 181             foo.set_uri(unicode_filename, self._bar_txt_uri)
 182
 183             _ign, n, baz_file = self.makefile(2)
 184             self._baz_file_uri = baz_file
 185             sub.set_uri(u"baz.txt", baz_file)
 186
 187             _ign, n, self._bad_file_uri = self.makefile(3)
 188             # this uri should not be downloadable
 189             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 190
 191             rodir = res[5][1]
 192             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 193             rodir.set_uri(u"nor", baz_file)
 194
 195             # public/
 196             # public/foo/
 197             # public/foo/bar.txt
 198             # public/foo/blockingfile
 199             # public/foo/empty/
 200             # public/foo/sub/
 201             # public/foo/sub/baz.txt
 202             # public/reedownlee/
 203             # public/reedownlee/nor
 204             self.NEWFILE_CONTENTS = "newfile contents\n"
 205
 206             return foo.get_metadata_for(u"bar.txt")
 207         d.addCallback(_then)
 208         def _got_metadata(metadata):
 209             self._bar_txt_metadata = metadata
 210         d.addCallback(_got_metadata)
 211         return d
 212
 213     def makefile(self, number):
 214         contents = "contents of file %s\n" % number
 215         n = create_chk_filenode(self.s, contents)
 216         return contents, n, n.get_uri()
 217
 218     def tearDown(self):
 219         return self.s.stopService()
 220
 221     def failUnlessIsBarDotTxt(self, res):
 222         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 223
 224     def failUnlessIsBarJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], u"filenode")
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failIf(data[1]["mutable"])
 230         self.failIf("rw_uri" in data[1]) # immutable
 231         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 232         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 233         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 234
 235     def failUnlessIsFooJSON(self, res):
 236         data = simplejson.loads(res)
 237         self.failUnless(isinstance(data, list))
 238         self.failUnlessEqual(data[0], "dirnode", res)
 239         self.failUnless(isinstance(data[1], dict))
 240         self.failUnless(data[1]["mutable"])
 241         self.failUnless("rw_uri" in data[1]) # mutable
 242         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 243         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 244         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 245
 246         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 247         self.failUnlessEqual(kidnames,
 248                              [u"bar.txt", u"blockingfile", u"empty",
 249                               u"n\u00fc.txt", u"sub"])
 250         kids = dict( [(unicode(name),value)
 251                       for (name,value)
 252                       in data[1]["children"].iteritems()] )
 253         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 254         self.failUnless("metadata" in kids[u"sub"][1])
 255         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 256         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 257         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 258         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 259         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 260         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 261                              self._bar_txt_verifycap)
 262         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 263                              self._bar_txt_metadata["ctime"])
 264         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 265                              self._bar_txt_uri)
 266
 267     def GET(self, urlpath, followRedirect=False, return_response=False,
 268             **kwargs):
 269         # if return_response=True, this fires with (data, statuscode,
 270         # respheaders) instead of just data.
 271         assert not isinstance(urlpath, unicode)
 272         url = self.webish_url + urlpath
 273         factory = HTTPClientGETFactory(url, method="GET",
 274                                        followRedirect=followRedirect, **kwargs)
 275         reactor.connectTCP("localhost", self.webish_port, factory)
 276         d = factory.deferred
 277         def _got_data(data):
 278             return (data, factory.status, factory.response_headers)
 279         if return_response:
 280             d.addCallback(_got_data)
 281         return factory.deferred
 282
 283     def HEAD(self, urlpath, return_response=False, **kwargs):
 284         # this requires some surgery, because twisted.web.client doesn't want
 285         # to give us back the response headers.
 286         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 287         reactor.connectTCP("localhost", self.webish_port, factory)
 288         d = factory.deferred
 289         def _got_data(data):
 290             return (data, factory.status, factory.response_headers)
 291         if return_response:
 292             d.addCallback(_got_data)
 293         return factory.deferred
 294
 295     def PUT(self, urlpath, data, **kwargs):
 296         url = self.webish_url + urlpath
 297         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 298
 299     def DELETE(self, urlpath):
 300         url = self.webish_url + urlpath
 301         return client.getPage(url, method="DELETE")
 302
 303     def POST(self, urlpath, followRedirect=False, **fields):
 304         url = self.webish_url + urlpath
 305         sepbase = "boogabooga"
 306         sep = "--" + sepbase
 307         form = []
 308         form.append(sep)
 309         form.append('Content-Disposition: form-data; name="_charset"')
 310         form.append('')
 311         form.append('UTF-8')
 312         form.append(sep)
 313         for name, value in fields.iteritems():
 314             if isinstance(value, tuple):
 315                 filename, value = value
 316                 form.append('Content-Disposition: form-data; name="%s"; '
 317                             'filename="%s"' % (name, filename.encode("utf-8")))
 318             else:
 319                 form.append('Content-Disposition: form-data; name="%s"' % name)
 320             form.append('')
 321             if isinstance(value, unicode):
 322                 value = value.encode("utf-8")
 323             else:
 324                 value = str(value)
 325             assert isinstance(value, str)
 326             form.append(value)
 327             form.append(sep)
 328         form[-1] += "--"
 329         body = "\r\n".join(form) + "\r\n"
 330         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 331                    }
 332         return client.getPage(url, method="POST", postdata=body,
 333                               headers=headers, followRedirect=followRedirect)
 334
 335     def shouldFail(self, res, expected_failure, which,
 336                    substring=None, response_substring=None):
 337         if isinstance(res, failure.Failure):
 338             res.trap(expected_failure)
 339             if substring:
 340                 self.failUnless(substring in str(res),
 341                                 "substring '%s' not in '%s'"
 342                                 % (substring, str(res)))
 343             if response_substring:
 344                 self.failUnless(response_substring in res.value.response,
 345                                 "response substring '%s' not in '%s'"
 346                                 % (response_substring, res.value.response))
 347         else:
 348             self.fail("%s was supposed to raise %s, not get '%s'" %
 349                       (which, expected_failure, res))
 350
 351     def shouldFail2(self, expected_failure, which, substring,
 352                     response_substring,
 353                     callable, *args, **kwargs):
 354         assert substring is None or isinstance(substring, str)
 355         assert response_substring is None or isinstance(response_substring, str)
 356         d = defer.maybeDeferred(callable, *args, **kwargs)
 357         def done(res):
 358             if isinstance(res, failure.Failure):
 359                 res.trap(expected_failure)
 360                 if substring:
 361                     self.failUnless(substring in str(res),
 362                                     "%s: substring '%s' not in '%s'"
 363                                     % (which, substring, str(res)))
 364                 if response_substring:
 365                     self.failUnless(response_substring in res.value.response,
 366                                     "%s: response substring '%s' not in '%s'"
 367                                     % (which,
 368                                        response_substring, res.value.response))
 369             else:
 370                 self.fail("%s was supposed to raise %s, not get '%s'" %
 371                           (which, expected_failure, res))
 372         d.addBoth(done)
 373         return d
 374
 375     def should404(self, res, which):
 376         if isinstance(res, failure.Failure):
 377             res.trap(error.Error)
 378             self.failUnlessEqual(res.value.status, "404")
 379         else:
 380             self.fail("%s was supposed to Error(404), not get '%s'" %
 381                       (which, res))
 382
 383     def shouldHTTPError(self, res, which, code=None, substring=None,
 384                         response_substring=None):
 385         if isinstance(res, failure.Failure):
 386             res.trap(error.Error)
 387             if code is not None:
 388                 self.failUnlessEqual(res.value.status, str(code))
 389             if substring:
 390                 self.failUnless(substring in str(res),
 391                                 "substring '%s' not in '%s'"
 392                                 % (substring, str(res)))
 393             if response_substring:
 394                 self.failUnless(response_substring in res.value.response,
 395                                 "response substring '%s' not in '%s'"
 396                                 % (response_substring, res.value.response))
 397         else:
 398             self.fail("%s was supposed to Error(%s), not get '%s'" %
 399                       (which, code, res))
 400
 401     def shouldHTTPError2(self, which,
 402                          code=None, substring=None, response_substring=None,
 403                          callable=None, *args, **kwargs):
 404         assert substring is None or isinstance(substring, str)
 405         assert callable
 406         d = defer.maybeDeferred(callable, *args, **kwargs)
 407         d.addBoth(self.shouldHTTPError, which,
 408                   code, substring, response_substring)
 409         return d
 410
 411
 412 class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
 413     def test_create(self):
 414         pass
 415
 416     def test_welcome(self):
 417         d = self.GET("/")
 418         def _check(res):
 419             self.failUnless('Welcome To AllMyData' in res)
 420             self.failUnless('Tahoe' in res)
 421
 422             self.s.basedir = 'web/test_welcome'
 423             fileutil.make_dirs("web/test_welcome")
 424             fileutil.make_dirs("web/test_welcome/private")
 425             return self.GET("/")
 426         d.addCallback(_check)
 427         return d
 428
 429     def test_provisioning(self):
 430         d = self.GET("/provisioning/")
 431         def _check(res):
 432             self.failUnless('Tahoe Provisioning Tool' in res)
 433             fields = {'filled': True,
 434                       "num_users": int(50e3),
 435                       "files_per_user": 1000,
 436                       "space_per_user": int(1e9),
 437                       "sharing_ratio": 1.0,
 438                       "encoding_parameters": "3-of-10-5",
 439                       "num_servers": 30,
 440                       "ownership_mode": "A",
 441                       "download_rate": 100,
 442                       "upload_rate": 10,
 443                       "delete_rate": 10,
 444                       "lease_timer": 7,
 445                       }
 446             return self.POST("/provisioning/", **fields)
 447
 448         d.addCallback(_check)
 449         def _check2(res):
 450             self.failUnless('Tahoe Provisioning Tool' in res)
 451             self.failUnless("Share space consumed: 167.01TB" in res)
 452
 453             fields = {'filled': True,
 454                       "num_users": int(50e6),
 455                       "files_per_user": 1000,
 456                       "space_per_user": int(5e9),
 457                       "sharing_ratio": 1.0,
 458                       "encoding_parameters": "25-of-100-50",
 459                       "num_servers": 30000,
 460                       "ownership_mode": "E",
 461                       "drive_failure_model": "U",
 462                       "drive_size": 1000,
 463                       "download_rate": 1000,
 464                       "upload_rate": 100,
 465                       "delete_rate": 100,
 466                       "lease_timer": 7,
 467                       }
 468             return self.POST("/provisioning/", **fields)
 469         d.addCallback(_check2)
 470         def _check3(res):
 471             self.failUnless("Share space consumed: huge!" in res)
 472             fields = {'filled': True}
 473             return self.POST("/provisioning/", **fields)
 474         d.addCallback(_check3)
 475         def _check4(res):
 476             self.failUnless("Share space consumed:" in res)
 477         d.addCallback(_check4)
 478         return d
 479
 480     def test_reliability_tool(self):
 481         try:
 482             from allmydata import reliability
 483             _hush_pyflakes = reliability
 484         except:
 485             raise unittest.SkipTest("reliability tool requires NumPy")
 486
 487         d = self.GET("/reliability/")
 488         def _check(res):
 489             self.failUnless('Tahoe Reliability Tool' in res)
 490             fields = {'drive_lifetime': "8Y",
 491                       "k": "3",
 492                       "R": "7",
 493                       "N": "10",
 494                       "delta": "100000",
 495                       "check_period": "1M",
 496                       "report_period": "3M",
 497                       "report_span": "5Y",
 498                       }
 499             return self.POST("/reliability/", **fields)
 500
 501         d.addCallback(_check)
 502         def _check2(res):
 503             self.failUnless('Tahoe Reliability Tool' in res)
 504             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 505             self.failUnless(re.search(r, res), res)
 506         d.addCallback(_check2)
 507         return d
 508
 509     def test_status(self):
 510         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 511         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 512         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 513         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 514         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 515         d = self.GET("/status", followRedirect=True)
 516         def _check(res):
 517             self.failUnless('Upload and Download Status' in res, res)
 518             self.failUnless('"down-%d"' % dl_num in res, res)
 519             self.failUnless('"up-%d"' % ul_num in res, res)
 520             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 521             self.failUnless('"publish-%d"' % pub_num in res, res)
 522             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 523         d.addCallback(_check)
 524         d.addCallback(lambda res: self.GET("/status/?t=json"))
 525         def _check_json(res):
 526             data = simplejson.loads(res)
 527             self.failUnless(isinstance(data, dict))
 528             active = data["active"]
 529             # TODO: test more. We need a way to fake an active operation
 530             # here.
 531         d.addCallback(_check_json)
 532
 533         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 534         def _check_dl(res):
 535             self.failUnless("File Download Status" in res, res)
 536         d.addCallback(_check_dl)
 537         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 538         def _check_ul(res):
 539             self.failUnless("File Upload Status" in res, res)
 540         d.addCallback(_check_ul)
 541         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 542         def _check_mapupdate(res):
 543             self.failUnless("Mutable File Servermap Update Status" in res, res)
 544         d.addCallback(_check_mapupdate)
 545         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 546         def _check_publish(res):
 547             self.failUnless("Mutable File Publish Status" in res, res)
 548         d.addCallback(_check_publish)
 549         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 550         def _check_retrieve(res):
 551             self.failUnless("Mutable File Retrieve Status" in res, res)
 552         d.addCallback(_check_retrieve)
 553
 554         return d
 555
 556     def test_status_numbers(self):
 557         drrm = status.DownloadResultsRendererMixin()
 558         self.failUnlessEqual(drrm.render_time(None, None), "")
 559         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 560         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 561         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 562         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 563         self.failUnlessEqual(drrm.render_rate(None, None), "")
 564         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 565         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 566         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 567
 568         urrm = status.UploadResultsRendererMixin()
 569         self.failUnlessEqual(urrm.render_time(None, None), "")
 570         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 571         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 572         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 573         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 574         self.failUnlessEqual(urrm.render_rate(None, None), "")
 575         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 576         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 577         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 578
 579     def test_GET_FILEURL(self):
 580         d = self.GET(self.public_url + "/foo/bar.txt")
 581         d.addCallback(self.failUnlessIsBarDotTxt)
 582         return d
 583
 584     def test_GET_FILEURL_range(self):
 585         headers = {"range": "bytes=1-10"}
 586         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 587                      return_response=True)
 588         def _got((res, status, headers)):
 589             self.failUnlessEqual(int(status), 206)
 590             self.failUnless(headers.has_key("content-range"))
 591             self.failUnlessEqual(headers["content-range"][0],
 592                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 593             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 594         d.addCallback(_got)
 595         return d
 596
 597     def test_GET_FILEURL_partial_range(self):
 598         headers = {"range": "bytes=5-"}
 599         length  = len(self.BAR_CONTENTS)
 600         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 601                      return_response=True)
 602         def _got((res, status, headers)):
 603             self.failUnlessEqual(int(status), 206)
 604             self.failUnless(headers.has_key("content-range"))
 605             self.failUnlessEqual(headers["content-range"][0],
 606                                  "bytes 5-%d/%d" % (length-1, length))
 607             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 608         d.addCallback(_got)
 609         return d
 610
 611     def test_HEAD_FILEURL_range(self):
 612         headers = {"range": "bytes=1-10"}
 613         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 614                      return_response=True)
 615         def _got((res, status, headers)):
 616             self.failUnlessEqual(res, "")
 617             self.failUnlessEqual(int(status), 206)
 618             self.failUnless(headers.has_key("content-range"))
 619             self.failUnlessEqual(headers["content-range"][0],
 620                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 621         d.addCallback(_got)
 622         return d
 623
 624     def test_HEAD_FILEURL_partial_range(self):
 625         headers = {"range": "bytes=5-"}
 626         length  = len(self.BAR_CONTENTS)
 627         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 628                      return_response=True)
 629         def _got((res, status, headers)):
 630             self.failUnlessEqual(int(status), 206)
 631             self.failUnless(headers.has_key("content-range"))
 632             self.failUnlessEqual(headers["content-range"][0],
 633                                  "bytes 5-%d/%d" % (length-1, length))
 634         d.addCallback(_got)
 635         return d
 636
 637     def test_GET_FILEURL_range_bad(self):
 638         headers = {"range": "BOGUS=fizbop-quarnak"}
 639         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 640                              "400 Bad Request",
 641                              "Syntactically invalid http range header",
 642                              self.GET, self.public_url + "/foo/bar.txt",
 643                              headers=headers)
 644         return d
 645
 646     def test_HEAD_FILEURL(self):
 647         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 648         def _got((res, status, headers)):
 649             self.failUnlessEqual(res, "")
 650             self.failUnlessEqual(headers["content-length"][0],
 651                                  str(len(self.BAR_CONTENTS)))
 652             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 653         d.addCallback(_got)
 654         return d
 655
 656     def test_GET_FILEURL_named(self):
 657         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 658         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 659         d = self.GET(base + "/@@name=/blah.txt")
 660         d.addCallback(self.failUnlessIsBarDotTxt)
 661         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 662         d.addCallback(self.failUnlessIsBarDotTxt)
 663         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 664         d.addCallback(self.failUnlessIsBarDotTxt)
 665         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 666         d.addCallback(self.failUnlessIsBarDotTxt)
 667         save_url = base + "?save=true&filename=blah.txt"
 668         d.addCallback(lambda res: self.GET(save_url))
 669         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 670         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 671         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 672         u_url = base + "?save=true&filename=" + u_fn_e
 673         d.addCallback(lambda res: self.GET(u_url))
 674         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 675         return d
 676
 677     def test_PUT_FILEURL_named_bad(self):
 678         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 679         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 680                              "400 Bad Request",
 681                              "/file can only be used with GET or HEAD",
 682                              self.PUT, base + "/@@name=/blah.txt", "")
 683         return d
 684
 685     def test_GET_DIRURL_named_bad(self):
 686         base = "/file/%s" % urllib.quote(self._foo_uri)
 687         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 688                              "400 Bad Request",
 689                              "is not a file-cap",
 690                              self.GET, base + "/@@name=/blah.txt")
 691         return d
 692
 693     def test_GET_slash_file_bad(self):
 694         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 695                              "404 Not Found",
 696                              "/file must be followed by a file-cap and a name",
 697                              self.GET, "/file")
 698         return d
 699
 700     def test_GET_unhandled_URI_named(self):
 701         contents, n, newuri = self.makefile(12)
 702         verifier_cap = n.get_verify_cap().to_string()
 703         base = "/file/%s" % urllib.quote(verifier_cap)
 704         # client.create_node_from_uri() can't handle verify-caps
 705         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 706                              "400 Bad Request",
 707                              "is not a valid file- or directory- cap",
 708                              self.GET, base)
 709         return d
 710
 711     def test_GET_unhandled_URI(self):
 712         contents, n, newuri = self.makefile(12)
 713         verifier_cap = n.get_verify_cap().to_string()
 714         base = "/uri/%s" % urllib.quote(verifier_cap)
 715         # client.create_node_from_uri() can't handle verify-caps
 716         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 717                              "400 Bad Request",
 718                              "is not a valid file- or directory- cap",
 719                              self.GET, base)
 720         return d
 721
 722     def test_GET_FILE_URI(self):
 723         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 724         d = self.GET(base)
 725         d.addCallback(self.failUnlessIsBarDotTxt)
 726         return d
 727
 728     def test_GET_FILE_URI_badchild(self):
 729         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 730         errmsg = "Files have no children, certainly not named 'boguschild'"
 731         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 732                              "400 Bad Request", errmsg,
 733                              self.GET, base)
 734         return d
 735
 736     def test_PUT_FILE_URI_badchild(self):
 737         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 738         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 739         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 740                              "400 Bad Request", errmsg,
 741                              self.PUT, base, "")
 742         return d
 743
 744     def test_GET_FILEURL_save(self):
 745         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 746         # TODO: look at the headers, expect a Content-Disposition: attachment
 747         # header.
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         return d
 750
 751     def test_GET_FILEURL_missing(self):
 752         d = self.GET(self.public_url + "/foo/missing")
 753         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 754         return d
 755
 756     def test_PUT_NEWFILEURL(self):
 757         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 758         # TODO: we lose the response code, so we can't check this
 759         #self.failUnlessEqual(responsecode, 201)
 760         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 761         d.addCallback(lambda res:
 762                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 763                                                       self.NEWFILE_CONTENTS))
 764         return d
 765
 766     def test_PUT_NEWFILEURL_range_bad(self):
 767         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 768         target = self.public_url + "/foo/new.txt"
 769         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 770                              "501 Not Implemented",
 771                              "Content-Range in PUT not yet supported",
 772                              # (and certainly not for immutable files)
 773                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 774                              headers=headers)
 775         d.addCallback(lambda res:
 776                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 777         return d
 778
 779     def test_PUT_NEWFILEURL_mutable(self):
 780         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 781                      self.NEWFILE_CONTENTS)
 782         # TODO: we lose the response code, so we can't check this
 783         #self.failUnlessEqual(responsecode, 201)
 784         def _check_uri(res):
 785             u = uri.from_string_mutable_filenode(res)
 786             self.failUnless(u.is_mutable())
 787             self.failIf(u.is_readonly())
 788             return res
 789         d.addCallback(_check_uri)
 790         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 791         d.addCallback(lambda res:
 792                       self.failUnlessMutableChildContentsAre(self._foo_node,
 793                                                              u"new.txt",
 794                                                              self.NEWFILE_CONTENTS))
 795         return d
 796
 797     def test_PUT_NEWFILEURL_mutable_toobig(self):
 798         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 799                              "413 Request Entity Too Large",
 800                              "SDMF is limited to one segment, and 10001 > 10000",
 801                              self.PUT,
 802                              self.public_url + "/foo/new.txt?mutable=true",
 803                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 804         return d
 805
 806     def test_PUT_NEWFILEURL_replace(self):
 807         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 808         # TODO: we lose the response code, so we can't check this
 809         #self.failUnlessEqual(responsecode, 200)
 810         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 811         d.addCallback(lambda res:
 812                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 813                                                       self.NEWFILE_CONTENTS))
 814         return d
 815
 816     def test_PUT_NEWFILEURL_bad_t(self):
 817         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 818                              "PUT to a file: bad t=bogus",
 819                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 820                              "contents")
 821         return d
 822
 823     def test_PUT_NEWFILEURL_no_replace(self):
 824         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 825                      self.NEWFILE_CONTENTS)
 826         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 827                   "409 Conflict",
 828                   "There was already a child by that name, and you asked me "
 829                   "to not replace it")
 830         return d
 831
 832     def test_PUT_NEWFILEURL_mkdirs(self):
 833         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 834         fn = self._foo_node
 835         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 836         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 837         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 838         d.addCallback(lambda res:
 839                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 840                                                       self.NEWFILE_CONTENTS))
 841         return d
 842
 843     def test_PUT_NEWFILEURL_blocked(self):
 844         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 845                      self.NEWFILE_CONTENTS)
 846         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 847                   "409 Conflict",
 848                   "Unable to create directory 'blockingfile': a file was in the way")
 849         return d
 850
 851     def test_DELETE_FILEURL(self):
 852         d = self.DELETE(self.public_url + "/foo/bar.txt")
 853         d.addCallback(lambda res:
 854                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 855         return d
 856
 857     def test_DELETE_FILEURL_missing(self):
 858         d = self.DELETE(self.public_url + "/foo/missing")
 859         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 860         return d
 861
 862     def test_DELETE_FILEURL_missing2(self):
 863         d = self.DELETE(self.public_url + "/missing/missing")
 864         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 865         return d
 866
 867     def test_GET_FILEURL_json(self):
 868         # twisted.web.http.parse_qs ignores any query args without an '=', so
 869         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 870         # instead. This may make it tricky to emulate the S3 interface
 871         # completely.
 872         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 873         d.addCallback(self.failUnlessIsBarJSON)
 874         return d
 875
 876     def test_GET_FILEURL_json_missing(self):
 877         d = self.GET(self.public_url + "/foo/missing?json")
 878         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 879         return d
 880
 881     def test_GET_FILEURL_uri(self):
 882         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 883         def _check(res):
 884             self.failUnlessEqual(res, self._bar_txt_uri)
 885         d.addCallback(_check)
 886         d.addCallback(lambda res:
 887                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 888         def _check2(res):
 889             # for now, for files, uris and readonly-uris are the same
 890             self.failUnlessEqual(res, self._bar_txt_uri)
 891         d.addCallback(_check2)
 892         return d
 893
 894     def test_GET_FILEURL_badtype(self):
 895         d = self.shouldHTTPError2("GET t=bogus", 400, "Bad Request",
 896                                   "bad t=bogus",
 897                                   self.GET,
 898                                   self.public_url + "/foo/bar.txt?t=bogus")
 899         return d
 900
 901     def test_GET_FILEURL_uri_missing(self):
 902         d = self.GET(self.public_url + "/foo/missing?t=uri")
 903         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 904         return d
 905
 906     def test_GET_DIRURL(self):
 907         # the addSlash means we get a redirect here
 908         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 909         ROOT = "../../.."
 910         d = self.GET(self.public_url + "/foo", followRedirect=True)
 911         def _check(res):
 912             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 913                             in res, res)
 914             # the FILE reference points to a URI, but it should end in bar.txt
 915             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 916                        (ROOT, urllib.quote(self._bar_txt_uri)))
 917             get_bar = "".join([r'<td>',
 918                                r'<a href="%s">bar.txt</a>' % bar_url,
 919                                r'</td>',
 920                                r'\s+<td>FILE</td>',
 921                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 922                                ])
 923             self.failUnless(re.search(get_bar, res), res)
 924             for line in res.split("\n"):
 925                 # find the line that contains the delete button for bar.txt
 926                 if ("form action" in line and
 927                     'value="delete"' in line and
 928                     'value="bar.txt"' in line):
 929                     # the form target should use a relative URL
 930                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 931                     self.failUnless(('action="%s"' % foo_url) in line, line)
 932                     # and the when_done= should too
 933                     #done_url = urllib.quote(???)
 934                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 935                     #                in line, line)
 936                     break
 937             else:
 938                 self.fail("unable to find delete-bar.txt line", res)
 939
 940             # the DIR reference just points to a URI
 941             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 942             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 943                        + r'\s+<td>DIR</td>')
 944             self.failUnless(re.search(get_sub, res), res)
 945         d.addCallback(_check)
 946
 947         # look at a directory which is readonly
 948         d.addCallback(lambda res:
 949                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 950         def _check2(res):
 951             self.failUnless("(readonly)" in res, res)
 952             self.failIf("Upload a file" in res, res)
 953         d.addCallback(_check2)
 954
 955         # and at a directory that contains a readonly directory
 956         d.addCallback(lambda res:
 957                       self.GET(self.public_url, followRedirect=True))
 958         def _check3(res):
 959             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 960                                       '</td>\s+<td>DIR-RO</td>', res))
 961         d.addCallback(_check3)
 962
 963         return d
 964
 965     def test_GET_DIRURL_badtype(self):
 966         d = self.shouldHTTPError2("test_GET_DIRURL_badtype",
 967                                   400, "Bad Request",
 968                                   "bad t=bogus",
 969                                   self.GET,
 970                                   self.public_url + "/foo?t=bogus")
 971         return d
 972
 973     def test_GET_DIRURL_json(self):
 974         d = self.GET(self.public_url + "/foo?t=json")
 975         d.addCallback(self.failUnlessIsFooJSON)
 976         return d
 977
 978
 979     def test_POST_DIRURL_manifest_no_ophandle(self):
 980         d = self.shouldFail2(error.Error,
 981                              "test_POST_DIRURL_manifest_no_ophandle",
 982                              "400 Bad Request",
 983                              "slow operation requires ophandle=",
 984                              self.POST, self.public_url, t="start-manifest")
 985         return d
 986
 987     def test_POST_DIRURL_manifest(self):
 988         d = defer.succeed(None)
 989         def getman(ignored, output):
 990             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 991                           followRedirect=True)
 992             d.addCallback(self.wait_for_operation, "125")
 993             d.addCallback(self.get_operation_results, "125", output)
 994             return d
 995         d.addCallback(getman, None)
 996         def _got_html(manifest):
 997             self.failUnless("Manifest of SI=" in manifest)
 998             self.failUnless("<td>sub</td>" in manifest)
 999             self.failUnless(self._sub_uri in manifest)
1000             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1001         d.addCallback(_got_html)
1002
1003         # both t=status and unadorned GET should be identical
1004         d.addCallback(lambda res: self.GET("/operations/125"))
1005         d.addCallback(_got_html)
1006
1007         d.addCallback(getman, "html")
1008         d.addCallback(_got_html)
1009         d.addCallback(getman, "text")
1010         def _got_text(manifest):
1011             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1012             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1013         d.addCallback(_got_text)
1014         d.addCallback(getman, "JSON")
1015         def _got_json(res):
1016             data = res["manifest"]
1017             got = {}
1018             for (path_list, cap) in data:
1019                 got[tuple(path_list)] = cap
1020             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1021             self.failUnless((u"sub",u"baz.txt") in got)
1022             self.failUnless("finished" in res)
1023             self.failUnless("origin" in res)
1024             self.failUnless("storage-index" in res)
1025             self.failUnless("verifycaps" in res)
1026             self.failUnless("stats" in res)
1027         d.addCallback(_got_json)
1028         return d
1029
1030     def test_POST_DIRURL_deepsize_no_ophandle(self):
1031         d = self.shouldFail2(error.Error,
1032                              "test_POST_DIRURL_deepsize_no_ophandle",
1033                              "400 Bad Request",
1034                              "slow operation requires ophandle=",
1035                              self.POST, self.public_url, t="start-deep-size")
1036         return d
1037
1038     def test_POST_DIRURL_deepsize(self):
1039         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1040                       followRedirect=True)
1041         d.addCallback(self.wait_for_operation, "126")
1042         d.addCallback(self.get_operation_results, "126", "json")
1043         def _got_json(data):
1044             self.failUnlessEqual(data["finished"], True)
1045             size = data["size"]
1046             self.failUnless(size > 1000)
1047         d.addCallback(_got_json)
1048         d.addCallback(self.get_operation_results, "126", "text")
1049         def _got_text(res):
1050             mo = re.search(r'^size: (\d+)$', res, re.M)
1051             self.failUnless(mo, res)
1052             size = int(mo.group(1))
1053             # with directories, the size varies.
1054             self.failUnless(size > 1000)
1055         d.addCallback(_got_text)
1056         return d
1057
1058     def test_POST_DIRURL_deepstats_no_ophandle(self):
1059         d = self.shouldFail2(error.Error,
1060                              "test_POST_DIRURL_deepstats_no_ophandle",
1061                              "400 Bad Request",
1062                              "slow operation requires ophandle=",
1063                              self.POST, self.public_url, t="start-deep-stats")
1064         return d
1065
1066     def test_POST_DIRURL_deepstats(self):
1067         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1068                       followRedirect=True)
1069         d.addCallback(self.wait_for_operation, "127")
1070         d.addCallback(self.get_operation_results, "127", "json")
1071         def _got_json(stats):
1072             expected = {"count-immutable-files": 3,
1073                         "count-mutable-files": 0,
1074                         "count-literal-files": 0,
1075                         "count-files": 3,
1076                         "count-directories": 3,
1077                         "size-immutable-files": 57,
1078                         "size-literal-files": 0,
1079                         #"size-directories": 1912, # varies
1080                         #"largest-directory": 1590,
1081                         "largest-directory-children": 5,
1082                         "largest-immutable-file": 19,
1083                         }
1084             for k,v in expected.iteritems():
1085                 self.failUnlessEqual(stats[k], v,
1086                                      "stats[%s] was %s, not %s" %
1087                                      (k, stats[k], v))
1088             self.failUnlessEqual(stats["size-files-histogram"],
1089                                  [ [11, 31, 3] ])
1090         d.addCallback(_got_json)
1091         return d
1092
1093     def test_POST_DIRURL_stream_manifest(self):
1094         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1095         def _check(res):
1096             self.failUnless(res.endswith("\n"))
1097             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1098             self.failUnlessEqual(len(units), 7)
1099             self.failUnlessEqual(units[-1]["type"], "stats")
1100             first = units[0]
1101             self.failUnlessEqual(first["path"], [])
1102             self.failUnlessEqual(first["cap"], self._foo_uri)
1103             self.failUnlessEqual(first["type"], "directory")
1104             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1105             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1106             self.failIfEqual(baz["storage-index"], None)
1107             self.failIfEqual(baz["verifycap"], None)
1108             self.failIfEqual(baz["repaircap"], None)
1109             return
1110         d.addCallback(_check)
1111         return d
1112
1113     def test_GET_DIRURL_uri(self):
1114         d = self.GET(self.public_url + "/foo?t=uri")
1115         def _check(res):
1116             self.failUnlessEqual(res, self._foo_uri)
1117         d.addCallback(_check)
1118         return d
1119
1120     def test_GET_DIRURL_readonly_uri(self):
1121         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1122         def _check(res):
1123             self.failUnlessEqual(res, self._foo_readonly_uri)
1124         d.addCallback(_check)
1125         return d
1126
1127     def test_PUT_NEWDIRURL(self):
1128         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1129         d.addCallback(lambda res:
1130                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1131         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1132         d.addCallback(self.failUnlessNodeKeysAre, [])
1133         return d
1134
1135     def test_PUT_NEWDIRURL_exists(self):
1136         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1137         d.addCallback(lambda res:
1138                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1139         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1140         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1141         return d
1142
1143     def test_PUT_NEWDIRURL_blocked(self):
1144         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1145                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1146                              self.PUT,
1147                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1148         d.addCallback(lambda res:
1149                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1150         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1151         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1152         return d
1153
1154     def test_PUT_NEWDIRURL_mkdir_p(self):
1155         d = defer.succeed(None)
1156         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1157         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1158         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1159         def mkdir_p(mkpnode):
1160             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1161             d = self.POST(url)
1162             def made_subsub(ssuri):
1163                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1164                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1165                 d = self.POST(url)
1166                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1167                 return d
1168             d.addCallback(made_subsub)
1169             return d
1170         d.addCallback(mkdir_p)
1171         return d
1172
1173     def test_PUT_NEWDIRURL_mkdirs(self):
1174         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1175         d.addCallback(lambda res:
1176                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1177         d.addCallback(lambda res:
1178                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1179         d.addCallback(lambda res:
1180                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1181         d.addCallback(self.failUnlessNodeKeysAre, [])
1182         return d
1183
1184     def test_DELETE_DIRURL(self):
1185         d = self.DELETE(self.public_url + "/foo")
1186         d.addCallback(lambda res:
1187                       self.failIfNodeHasChild(self.public_root, u"foo"))
1188         return d
1189
1190     def test_DELETE_DIRURL_missing(self):
1191         d = self.DELETE(self.public_url + "/foo/missing")
1192         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1193         d.addCallback(lambda res:
1194                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1195         return d
1196
1197     def test_DELETE_DIRURL_missing2(self):
1198         d = self.DELETE(self.public_url + "/missing")
1199         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1200         return d
1201
1202     def dump_root(self):
1203         print "NODEWALK"
1204         w = webish.DirnodeWalkerMixin()
1205         def visitor(childpath, childnode, metadata):
1206             print childpath
1207         d = w.walk(self.public_root, visitor)
1208         return d
1209
1210     def failUnlessNodeKeysAre(self, node, expected_keys):
1211         for k in expected_keys:
1212             assert isinstance(k, unicode)
1213         d = node.list()
1214         def _check(children):
1215             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1216         d.addCallback(_check)
1217         return d
1218     def failUnlessNodeHasChild(self, node, name):
1219         assert isinstance(name, unicode)
1220         d = node.list()
1221         def _check(children):
1222             self.failUnless(name in children)
1223         d.addCallback(_check)
1224         return d
1225     def failIfNodeHasChild(self, node, name):
1226         assert isinstance(name, unicode)
1227         d = node.list()
1228         def _check(children):
1229             self.failIf(name in children)
1230         d.addCallback(_check)
1231         return d
1232
1233     def failUnlessChildContentsAre(self, node, name, expected_contents):
1234         assert isinstance(name, unicode)
1235         d = node.get_child_at_path(name)
1236         d.addCallback(lambda node: node.download_to_data())
1237         def _check(contents):
1238             self.failUnlessEqual(contents, expected_contents)
1239         d.addCallback(_check)
1240         return d
1241
1242     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1243         assert isinstance(name, unicode)
1244         d = node.get_child_at_path(name)
1245         d.addCallback(lambda node: node.download_best_version())
1246         def _check(contents):
1247             self.failUnlessEqual(contents, expected_contents)
1248         d.addCallback(_check)
1249         return d
1250
1251     def failUnlessChildURIIs(self, node, name, expected_uri):
1252         assert isinstance(name, unicode)
1253         d = node.get_child_at_path(name)
1254         def _check(child):
1255             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1256         d.addCallback(_check)
1257         return d
1258
1259     def failUnlessURIMatchesChild(self, got_uri, node, name):
1260         assert isinstance(name, unicode)
1261         d = node.get_child_at_path(name)
1262         def _check(child):
1263             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1264         d.addCallback(_check)
1265         return d
1266
1267     def failUnlessCHKURIHasContents(self, got_uri, contents):
1268         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1269
1270     def test_POST_upload(self):
1271         d = self.POST(self.public_url + "/foo", t="upload",
1272                       file=("new.txt", self.NEWFILE_CONTENTS))
1273         fn = self._foo_node
1274         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1275         d.addCallback(lambda res:
1276                       self.failUnlessChildContentsAre(fn, u"new.txt",
1277                                                       self.NEWFILE_CONTENTS))
1278         return d
1279
1280     def test_POST_upload_unicode(self):
1281         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1282         d = self.POST(self.public_url + "/foo", t="upload",
1283                       file=(filename, self.NEWFILE_CONTENTS))
1284         fn = self._foo_node
1285         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1286         d.addCallback(lambda res:
1287                       self.failUnlessChildContentsAre(fn, filename,
1288                                                       self.NEWFILE_CONTENTS))
1289         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1290         d.addCallback(lambda res: self.GET(target_url))
1291         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1292                                                             self.NEWFILE_CONTENTS,
1293                                                             contents))
1294         return d
1295
1296     def test_POST_upload_unicode_named(self):
1297         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1298         d = self.POST(self.public_url + "/foo", t="upload",
1299                       name=filename,
1300                       file=("overridden", self.NEWFILE_CONTENTS))
1301         fn = self._foo_node
1302         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1303         d.addCallback(lambda res:
1304                       self.failUnlessChildContentsAre(fn, filename,
1305                                                       self.NEWFILE_CONTENTS))
1306         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1307         d.addCallback(lambda res: self.GET(target_url))
1308         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1309                                                             self.NEWFILE_CONTENTS,
1310                                                             contents))
1311         return d
1312
1313     def test_POST_upload_no_link(self):
1314         d = self.POST("/uri", t="upload",
1315                       file=("new.txt", self.NEWFILE_CONTENTS))
1316         def _check_upload_results(page):
1317             # this should be a page which describes the results of the upload
1318             # that just finished.
1319             self.failUnless("Upload Results:" in page)
1320             self.failUnless("URI:" in page)
1321             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1322             mo = uri_re.search(page)
1323             self.failUnless(mo, page)
1324             new_uri = mo.group(1)
1325             return new_uri
1326         d.addCallback(_check_upload_results)
1327         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1328         return d
1329
1330     def test_POST_upload_no_link_whendone(self):
1331         d = self.POST("/uri", t="upload", when_done="/",
1332                       file=("new.txt", self.NEWFILE_CONTENTS))
1333         d.addBoth(self.shouldRedirect, "/")
1334         return d
1335
1336     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1337         d = defer.maybeDeferred(callable, *args, **kwargs)
1338         def done(res):
1339             if isinstance(res, failure.Failure):
1340                 res.trap(error.PageRedirect)
1341                 statuscode = res.value.status
1342                 target = res.value.location
1343                 return checker(statuscode, target)
1344             self.fail("%s: callable was supposed to redirect, not return '%s'"
1345                       % (which, res))
1346         d.addBoth(done)
1347         return d
1348
1349     def test_POST_upload_no_link_whendone_results(self):
1350         def check(statuscode, target):
1351             self.failUnlessEqual(statuscode, str(http.FOUND))
1352             self.failUnless(target.startswith(self.webish_url), target)
1353             return client.getPage(target, method="GET")
1354         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1355                                  check,
1356                                  self.POST, "/uri", t="upload",
1357                                  when_done="/uri/%(uri)s",
1358                                  file=("new.txt", self.NEWFILE_CONTENTS))
1359         d.addCallback(lambda res:
1360                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1361         return d
1362
1363     def test_POST_upload_no_link_mutable(self):
1364         d = self.POST("/uri", t="upload", mutable="true",
1365                       file=("new.txt", self.NEWFILE_CONTENTS))
1366         def _check(new_uri):
1367             new_uri = new_uri.strip()
1368             self.new_uri = new_uri
1369             u = IURI(new_uri)
1370             self.failUnless(IMutableFileURI.providedBy(u))
1371             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1372             n = self.s.create_node_from_uri(new_uri)
1373             return n.download_best_version()
1374         d.addCallback(_check)
1375         def _check2(data):
1376             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1377             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1378         d.addCallback(_check2)
1379         def _check3(data):
1380             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1381             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1382         d.addCallback(_check3)
1383         def _check4(data):
1384             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1385         d.addCallback(_check4)
1386         return d
1387
1388     def test_POST_upload_no_link_mutable_toobig(self):
1389         d = self.shouldFail2(error.Error,
1390                              "test_POST_upload_no_link_mutable_toobig",
1391                              "413 Request Entity Too Large",
1392                              "SDMF is limited to one segment, and 10001 > 10000",
1393                              self.POST,
1394                              "/uri", t="upload", mutable="true",
1395                              file=("new.txt",
1396                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1397         return d
1398
1399     def test_POST_upload_mutable(self):
1400         # this creates a mutable file
1401         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1402                       file=("new.txt", self.NEWFILE_CONTENTS))
1403         fn = self._foo_node
1404         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1405         d.addCallback(lambda res:
1406                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1407                                                              self.NEWFILE_CONTENTS))
1408         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1409         def _got(newnode):
1410             self.failUnless(IMutableFileNode.providedBy(newnode))
1411             self.failUnless(newnode.is_mutable())
1412             self.failIf(newnode.is_readonly())
1413             self._mutable_node = newnode
1414             self._mutable_uri = newnode.get_uri()
1415         d.addCallback(_got)
1416
1417         # now upload it again and make sure that the URI doesn't change
1418         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1419         d.addCallback(lambda res:
1420                       self.POST(self.public_url + "/foo", t="upload",
1421                                 mutable="true",
1422                                 file=("new.txt", NEWER_CONTENTS)))
1423         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1424         d.addCallback(lambda res:
1425                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1426                                                              NEWER_CONTENTS))
1427         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1428         def _got2(newnode):
1429             self.failUnless(IMutableFileNode.providedBy(newnode))
1430             self.failUnless(newnode.is_mutable())
1431             self.failIf(newnode.is_readonly())
1432             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1433         d.addCallback(_got2)
1434
1435         # upload a second time, using PUT instead of POST
1436         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1437         d.addCallback(lambda res:
1438                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1439         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1440         d.addCallback(lambda res:
1441                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1442                                                              NEW2_CONTENTS))
1443
1444         # finally list the directory, since mutable files are displayed
1445         # slightly differently
1446
1447         d.addCallback(lambda res:
1448                       self.GET(self.public_url + "/foo/",
1449                                followRedirect=True))
1450         def _check_page(res):
1451             # TODO: assert more about the contents
1452             self.failUnless("SSK" in res)
1453             return res
1454         d.addCallback(_check_page)
1455
1456         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1457         def _got3(newnode):
1458             self.failUnless(IMutableFileNode.providedBy(newnode))
1459             self.failUnless(newnode.is_mutable())
1460             self.failIf(newnode.is_readonly())
1461             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1462         d.addCallback(_got3)
1463
1464         # look at the JSON form of the enclosing directory
1465         d.addCallback(lambda res:
1466                       self.GET(self.public_url + "/foo/?t=json",
1467                                followRedirect=True))
1468         def _check_page_json(res):
1469             parsed = simplejson.loads(res)
1470             self.failUnlessEqual(parsed[0], "dirnode")
1471             children = dict( [(unicode(name),value)
1472                               for (name,value)
1473                               in parsed[1]["children"].iteritems()] )
1474             self.failUnless("new.txt" in children)
1475             new_json = children["new.txt"]
1476             self.failUnlessEqual(new_json[0], "filenode")
1477             self.failUnless(new_json[1]["mutable"])
1478             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1479             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1480             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1481         d.addCallback(_check_page_json)
1482
1483         # and the JSON form of the file
1484         d.addCallback(lambda res:
1485                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1486         def _check_file_json(res):
1487             parsed = simplejson.loads(res)
1488             self.failUnlessEqual(parsed[0], "filenode")
1489             self.failUnless(parsed[1]["mutable"])
1490             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1491             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1492             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1493         d.addCallback(_check_file_json)
1494
1495         # and look at t=uri and t=readonly-uri
1496         d.addCallback(lambda res:
1497                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1498         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1499         d.addCallback(lambda res:
1500                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1501         def _check_ro_uri(res):
1502             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1503             self.failUnlessEqual(res, ro_uri)
1504         d.addCallback(_check_ro_uri)
1505
1506         # make sure we can get to it from /uri/URI
1507         d.addCallback(lambda res:
1508                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1509         d.addCallback(lambda res:
1510                       self.failUnlessEqual(res, NEW2_CONTENTS))
1511
1512         # and that HEAD computes the size correctly
1513         d.addCallback(lambda res:
1514                       self.HEAD(self.public_url + "/foo/new.txt",
1515                                 return_response=True))
1516         def _got_headers((res, status, headers)):
1517             self.failUnlessEqual(res, "")
1518             self.failUnlessEqual(headers["content-length"][0],
1519                                  str(len(NEW2_CONTENTS)))
1520             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1521         d.addCallback(_got_headers)
1522
1523         # make sure that size errors are displayed correctly for overwrite
1524         d.addCallback(lambda res:
1525                       self.shouldFail2(error.Error,
1526                                        "test_POST_upload_mutable-toobig",
1527                                        "413 Request Entity Too Large",
1528                                        "SDMF is limited to one segment, and 10001 > 10000",
1529                                        self.POST,
1530                                        self.public_url + "/foo", t="upload",
1531                                        mutable="true",
1532                                        file=("new.txt",
1533                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1534                                        ))
1535
1536         d.addErrback(self.dump_error)
1537         return d
1538
1539     def test_POST_upload_mutable_toobig(self):
1540         d = self.shouldFail2(error.Error,
1541                              "test_POST_upload_no_link_mutable_toobig",
1542                              "413 Request Entity Too Large",
1543                              "SDMF is limited to one segment, and 10001 > 10000",
1544                              self.POST,
1545                              self.public_url + "/foo",
1546                              t="upload", mutable="true",
1547                              file=("new.txt",
1548                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1549         return d
1550
1551     def dump_error(self, f):
1552         # if the web server returns an error code (like 400 Bad Request),
1553         # web.client.getPage puts the HTTP response body into the .response
1554         # attribute of the exception object that it gives back. It does not
1555         # appear in the Failure's repr(), so the ERROR that trial displays
1556         # will be rather terse and unhelpful. addErrback this method to the
1557         # end of your chain to get more information out of these errors.
1558         if f.check(error.Error):
1559             print "web.error.Error:"
1560             print f
1561             print f.value.response
1562         return f
1563
1564     def test_POST_upload_replace(self):
1565         d = self.POST(self.public_url + "/foo", t="upload",
1566                       file=("bar.txt", self.NEWFILE_CONTENTS))
1567         fn = self._foo_node
1568         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1569         d.addCallback(lambda res:
1570                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1571                                                       self.NEWFILE_CONTENTS))
1572         return d
1573
1574     def test_POST_upload_no_replace_ok(self):
1575         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1576                       file=("new.txt", self.NEWFILE_CONTENTS))
1577         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1578         d.addCallback(lambda res: self.failUnlessEqual(res,
1579                                                        self.NEWFILE_CONTENTS))
1580         return d
1581
1582     def test_POST_upload_no_replace_queryarg(self):
1583         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1584                       file=("bar.txt", self.NEWFILE_CONTENTS))
1585         d.addBoth(self.shouldFail, error.Error,
1586                   "POST_upload_no_replace_queryarg",
1587                   "409 Conflict",
1588                   "There was already a child by that name, and you asked me "
1589                   "to not replace it")
1590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1591         d.addCallback(self.failUnlessIsBarDotTxt)
1592         return d
1593
1594     def test_POST_upload_no_replace_field(self):
1595         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1596                       file=("bar.txt", self.NEWFILE_CONTENTS))
1597         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1598                   "409 Conflict",
1599                   "There was already a child by that name, and you asked me "
1600                   "to not replace it")
1601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1602         d.addCallback(self.failUnlessIsBarDotTxt)
1603         return d
1604
1605     def test_POST_upload_whendone(self):
1606         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1607                       file=("new.txt", self.NEWFILE_CONTENTS))
1608         d.addBoth(self.shouldRedirect, "/THERE")
1609         fn = self._foo_node
1610         d.addCallback(lambda res:
1611                       self.failUnlessChildContentsAre(fn, u"new.txt",
1612                                                       self.NEWFILE_CONTENTS))
1613         return d
1614
1615     def test_POST_upload_named(self):
1616         fn = self._foo_node
1617         d = self.POST(self.public_url + "/foo", t="upload",
1618                       name="new.txt", file=self.NEWFILE_CONTENTS)
1619         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1620         d.addCallback(lambda res:
1621                       self.failUnlessChildContentsAre(fn, u"new.txt",
1622                                                       self.NEWFILE_CONTENTS))
1623         return d
1624
1625     def test_POST_upload_named_badfilename(self):
1626         d = self.POST(self.public_url + "/foo", t="upload",
1627                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1628         d.addBoth(self.shouldFail, error.Error,
1629                   "test_POST_upload_named_badfilename",
1630                   "400 Bad Request",
1631                   "name= may not contain a slash",
1632                   )
1633         # make sure that nothing was added
1634         d.addCallback(lambda res:
1635                       self.failUnlessNodeKeysAre(self._foo_node,
1636                                                  [u"bar.txt", u"blockingfile",
1637                                                   u"empty", u"n\u00fc.txt",
1638                                                   u"sub"]))
1639         return d
1640
1641     def test_POST_FILEURL_check(self):
1642         bar_url = self.public_url + "/foo/bar.txt"
1643         d = self.POST(bar_url, t="check")
1644         def _check(res):
1645             self.failUnless("Healthy :" in res)
1646         d.addCallback(_check)
1647         redir_url = "http://allmydata.org/TARGET"
1648         def _check2(statuscode, target):
1649             self.failUnlessEqual(statuscode, str(http.FOUND))
1650             self.failUnlessEqual(target, redir_url)
1651         d.addCallback(lambda res:
1652                       self.shouldRedirect2("test_POST_FILEURL_check",
1653                                            _check2,
1654                                            self.POST, bar_url,
1655                                            t="check",
1656                                            when_done=redir_url))
1657         d.addCallback(lambda res:
1658                       self.POST(bar_url, t="check", return_to=redir_url))
1659         def _check3(res):
1660             self.failUnless("Healthy :" in res)
1661             self.failUnless("Return to parent directory" in res)
1662             self.failUnless(redir_url in res)
1663         d.addCallback(_check3)
1664
1665         d.addCallback(lambda res:
1666                       self.POST(bar_url, t="check", output="JSON"))
1667         def _check_json(res):
1668             data = simplejson.loads(res)
1669             self.failUnless("storage-index" in data)
1670             self.failUnless(data["results"]["healthy"])
1671         d.addCallback(_check_json)
1672
1673         return d
1674
1675     def test_POST_FILEURL_check_and_repair(self):
1676         bar_url = self.public_url + "/foo/bar.txt"
1677         d = self.POST(bar_url, t="check", repair="true")
1678         def _check(res):
1679             self.failUnless("Healthy :" in res)
1680         d.addCallback(_check)
1681         redir_url = "http://allmydata.org/TARGET"
1682         def _check2(statuscode, target):
1683             self.failUnlessEqual(statuscode, str(http.FOUND))
1684             self.failUnlessEqual(target, redir_url)
1685         d.addCallback(lambda res:
1686                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1687                                            _check2,
1688                                            self.POST, bar_url,
1689                                            t="check", repair="true",
1690                                            when_done=redir_url))
1691         d.addCallback(lambda res:
1692                       self.POST(bar_url, t="check", return_to=redir_url))
1693         def _check3(res):
1694             self.failUnless("Healthy :" in res)
1695             self.failUnless("Return to parent directory" in res)
1696             self.failUnless(redir_url in res)
1697         d.addCallback(_check3)
1698         return d
1699
1700     def test_POST_DIRURL_check(self):
1701         foo_url = self.public_url + "/foo/"
1702         d = self.POST(foo_url, t="check")
1703         def _check(res):
1704             self.failUnless("Healthy :" in res, res)
1705         d.addCallback(_check)
1706         redir_url = "http://allmydata.org/TARGET"
1707         def _check2(statuscode, target):
1708             self.failUnlessEqual(statuscode, str(http.FOUND))
1709             self.failUnlessEqual(target, redir_url)
1710         d.addCallback(lambda res:
1711                       self.shouldRedirect2("test_POST_DIRURL_check",
1712                                            _check2,
1713                                            self.POST, foo_url,
1714                                            t="check",
1715                                            when_done=redir_url))
1716         d.addCallback(lambda res:
1717                       self.POST(foo_url, t="check", return_to=redir_url))
1718         def _check3(res):
1719             self.failUnless("Healthy :" in res, res)
1720             self.failUnless("Return to parent directory" in res)
1721             self.failUnless(redir_url in res)
1722         d.addCallback(_check3)
1723
1724         d.addCallback(lambda res:
1725                       self.POST(foo_url, t="check", output="JSON"))
1726         def _check_json(res):
1727             data = simplejson.loads(res)
1728             self.failUnless("storage-index" in data)
1729             self.failUnless(data["results"]["healthy"])
1730         d.addCallback(_check_json)
1731
1732         return d
1733
1734     def test_POST_DIRURL_check_and_repair(self):
1735         foo_url = self.public_url + "/foo/"
1736         d = self.POST(foo_url, t="check", repair="true")
1737         def _check(res):
1738             self.failUnless("Healthy :" in res, res)
1739         d.addCallback(_check)
1740         redir_url = "http://allmydata.org/TARGET"
1741         def _check2(statuscode, target):
1742             self.failUnlessEqual(statuscode, str(http.FOUND))
1743             self.failUnlessEqual(target, redir_url)
1744         d.addCallback(lambda res:
1745                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1746                                            _check2,
1747                                            self.POST, foo_url,
1748                                            t="check", repair="true",
1749                                            when_done=redir_url))
1750         d.addCallback(lambda res:
1751                       self.POST(foo_url, t="check", return_to=redir_url))
1752         def _check3(res):
1753             self.failUnless("Healthy :" in res)
1754             self.failUnless("Return to parent directory" in res)
1755             self.failUnless(redir_url in res)
1756         d.addCallback(_check3)
1757         return d
1758
1759     def wait_for_operation(self, ignored, ophandle):
1760         url = "/operations/" + ophandle
1761         url += "?t=status&output=JSON"
1762         d = self.GET(url)
1763         def _got(res):
1764             data = simplejson.loads(res)
1765             if not data["finished"]:
1766                 d = self.stall(delay=1.0)
1767                 d.addCallback(self.wait_for_operation, ophandle)
1768                 return d
1769             return data
1770         d.addCallback(_got)
1771         return d
1772
1773     def get_operation_results(self, ignored, ophandle, output=None):
1774         url = "/operations/" + ophandle
1775         url += "?t=status"
1776         if output:
1777             url += "&output=" + output
1778         d = self.GET(url)
1779         def _got(res):
1780             if output and output.lower() == "json":
1781                 return simplejson.loads(res)
1782             return res
1783         d.addCallback(_got)
1784         return d
1785
1786     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1787         d = self.shouldFail2(error.Error,
1788                              "test_POST_DIRURL_deepcheck_no_ophandle",
1789                              "400 Bad Request",
1790                              "slow operation requires ophandle=",
1791                              self.POST, self.public_url, t="start-deep-check")
1792         return d
1793
1794     def test_POST_DIRURL_deepcheck(self):
1795         def _check_redirect(statuscode, target):
1796             self.failUnlessEqual(statuscode, str(http.FOUND))
1797             self.failUnless(target.endswith("/operations/123"))
1798         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1799                                  self.POST, self.public_url,
1800                                  t="start-deep-check", ophandle="123")
1801         d.addCallback(self.wait_for_operation, "123")
1802         def _check_json(data):
1803             self.failUnlessEqual(data["finished"], True)
1804             self.failUnlessEqual(data["count-objects-checked"], 8)
1805             self.failUnlessEqual(data["count-objects-healthy"], 8)
1806         d.addCallback(_check_json)
1807         d.addCallback(self.get_operation_results, "123", "html")
1808         def _check_html(res):
1809             self.failUnless("Objects Checked: <span>8</span>" in res)
1810             self.failUnless("Objects Healthy: <span>8</span>" in res)
1811         d.addCallback(_check_html)
1812
1813         d.addCallback(lambda res:
1814                       self.GET("/operations/123/"))
1815         d.addCallback(_check_html) # should be the same as without the slash
1816
1817         d.addCallback(lambda res:
1818                       self.shouldFail2(error.Error, "one", "404 Not Found",
1819                                        "No detailed results for SI bogus",
1820                                        self.GET, "/operations/123/bogus"))
1821
1822         foo_si = self._foo_node.get_storage_index()
1823         foo_si_s = base32.b2a(foo_si)
1824         d.addCallback(lambda res:
1825                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1826         def _check_foo_json(res):
1827             data = simplejson.loads(res)
1828             self.failUnlessEqual(data["storage-index"], foo_si_s)
1829             self.failUnless(data["results"]["healthy"])
1830         d.addCallback(_check_foo_json)
1831         return d
1832
1833     def test_POST_DIRURL_deepcheck_and_repair(self):
1834         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1835                       ophandle="124", output="json", followRedirect=True)
1836         d.addCallback(self.wait_for_operation, "124")
1837         def _check_json(data):
1838             self.failUnlessEqual(data["finished"], True)
1839             self.failUnlessEqual(data["count-objects-checked"], 8)
1840             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1841             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1842             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1843             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1844             self.failUnlessEqual(data["count-repairs-successful"], 0)
1845             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1846             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1847             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1848             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1849         d.addCallback(_check_json)
1850         d.addCallback(self.get_operation_results, "124", "html")
1851         def _check_html(res):
1852             self.failUnless("Objects Checked: <span>8</span>" in res)
1853
1854             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1855             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1856             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1857
1858             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1859             self.failUnless("Repairs Successful: <span>0</span>" in res)
1860             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1861
1862             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1863             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1864             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1865         d.addCallback(_check_html)
1866         return d
1867
1868     def test_POST_FILEURL_bad_t(self):
1869         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1870                              "POST to file: bad t=bogus",
1871                              self.POST, self.public_url + "/foo/bar.txt",
1872                              t="bogus")
1873         return d
1874
1875     def test_POST_mkdir(self): # return value?
1876         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1877         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1878         d.addCallback(self.failUnlessNodeKeysAre, [])
1879         return d
1880
1881     def test_POST_mkdir_2(self):
1882         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1883         d.addCallback(lambda res:
1884                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1885         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1886         d.addCallback(self.failUnlessNodeKeysAre, [])
1887         return d
1888
1889     def test_POST_mkdirs_2(self):
1890         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1891         d.addCallback(lambda res:
1892                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1893         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1894         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1895         d.addCallback(self.failUnlessNodeKeysAre, [])
1896         return d
1897
1898     def test_POST_mkdir_no_parentdir_noredirect(self):
1899         d = self.POST("/uri?t=mkdir")
1900         def _after_mkdir(res):
1901             uri.NewDirectoryURI.init_from_string(res)
1902         d.addCallback(_after_mkdir)
1903         return d
1904
1905     def test_POST_mkdir_no_parentdir_redirect(self):
1906         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1907         d.addBoth(self.shouldRedirect, None, statuscode='303')
1908         def _check_target(target):
1909             target = urllib.unquote(target)
1910             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1911         d.addCallback(_check_target)
1912         return d
1913
1914     def test_POST_noparent_bad(self):
1915         d = self.shouldHTTPError2("POST /uri?t=bogus", 400, "Bad Request",
1916                                   "/uri accepts only PUT, PUT?t=mkdir, "
1917                                   "POST?t=upload, and POST?t=mkdir",
1918                                   self.POST, "/uri?t=bogus")
1919         return d
1920
1921     def test_welcome_page_mkdir_button(self):
1922         # Fetch the welcome page.
1923         d = self.GET("/")
1924         def _after_get_welcome_page(res):
1925             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1926             mo = MKDIR_BUTTON_RE.search(res)
1927             formaction = mo.group(1)
1928             formt = mo.group(2)
1929             formaname = mo.group(3)
1930             formavalue = mo.group(4)
1931             return (formaction, formt, formaname, formavalue)
1932         d.addCallback(_after_get_welcome_page)
1933         def _after_parse_form(res):
1934             (formaction, formt, formaname, formavalue) = res
1935             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1936         d.addCallback(_after_parse_form)
1937         d.addBoth(self.shouldRedirect, None, statuscode='303')
1938         return d
1939
1940     def test_POST_mkdir_replace(self): # return value?
1941         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1942         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1943         d.addCallback(self.failUnlessNodeKeysAre, [])
1944         return d
1945
1946     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1947         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1948         d.addBoth(self.shouldFail, error.Error,
1949                   "POST_mkdir_no_replace_queryarg",
1950                   "409 Conflict",
1951                   "There was already a child by that name, and you asked me "
1952                   "to not replace it")
1953         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1954         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1955         return d
1956
1957     def test_POST_mkdir_no_replace_field(self): # return value?
1958         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1959                       replace="false")
1960         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1961                   "409 Conflict",
1962                   "There was already a child by that name, and you asked me "
1963                   "to not replace it")
1964         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1965         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1966         return d
1967
1968     def test_POST_mkdir_whendone_field(self):
1969         d = self.POST(self.public_url + "/foo",
1970                       t="mkdir", name="newdir", when_done="/THERE")
1971         d.addBoth(self.shouldRedirect, "/THERE")
1972         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1973         d.addCallback(self.failUnlessNodeKeysAre, [])
1974         return d
1975
1976     def test_POST_mkdir_whendone_queryarg(self):
1977         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1978                       t="mkdir", name="newdir")
1979         d.addBoth(self.shouldRedirect, "/THERE")
1980         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1981         d.addCallback(self.failUnlessNodeKeysAre, [])
1982         return d
1983
1984     def test_POST_bad_t(self):
1985         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1986                              "POST to a directory with bad t=BOGUS",
1987                              self.POST, self.public_url + "/foo", t="BOGUS")
1988         return d
1989
1990     def test_POST_set_children(self):
1991         contents9, n9, newuri9 = self.makefile(9)
1992         contents10, n10, newuri10 = self.makefile(10)
1993         contents11, n11, newuri11 = self.makefile(11)
1994
1995         reqbody = """{
1996                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1997                                                 "size": 0,
1998                                                 "metadata": {
1999                                                   "ctime": 1002777696.7564139,
2000                                                   "mtime": 1002777696.7564139
2001                                                  }
2002                                                } ],
2003                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2004                                                 "size": 1,
2005                                                 "metadata": {
2006                                                   "ctime": 1002777696.7564139,
2007                                                   "mtime": 1002777696.7564139
2008                                                  }
2009                                                } ],
2010                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2011                                                 "size": 2,
2012                                                 "metadata": {
2013                                                   "ctime": 1002777696.7564139,
2014                                                   "mtime": 1002777696.7564139
2015                                                  }
2016                                                } ]
2017                     }""" % (newuri9, newuri10, newuri11)
2018
2019         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2020
2021         d = client.getPage(url, method="POST", postdata=reqbody)
2022         def _then(res):
2023             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2024             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2025             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2026
2027         d.addCallback(_then)
2028         d.addErrback(self.dump_error)
2029         return d
2030
2031     def test_POST_put_uri(self):
2032         contents, n, newuri = self.makefile(8)
2033         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2034         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2035         d.addCallback(lambda res:
2036                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2037                                                       contents))
2038         return d
2039
2040     def test_POST_put_uri_replace(self):
2041         contents, n, newuri = self.makefile(8)
2042         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2043         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2044         d.addCallback(lambda res:
2045                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2046                                                       contents))
2047         return d
2048
2049     def test_POST_put_uri_no_replace_queryarg(self):
2050         contents, n, newuri = self.makefile(8)
2051         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2052                       name="bar.txt", uri=newuri)
2053         d.addBoth(self.shouldFail, error.Error,
2054                   "POST_put_uri_no_replace_queryarg",
2055                   "409 Conflict",
2056                   "There was already a child by that name, and you asked me "
2057                   "to not replace it")
2058         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2059         d.addCallback(self.failUnlessIsBarDotTxt)
2060         return d
2061
2062     def test_POST_put_uri_no_replace_field(self):
2063         contents, n, newuri = self.makefile(8)
2064         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2065                       name="bar.txt", uri=newuri)
2066         d.addBoth(self.shouldFail, error.Error,
2067                   "POST_put_uri_no_replace_field",
2068                   "409 Conflict",
2069                   "There was already a child by that name, and you asked me "
2070                   "to not replace it")
2071         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2072         d.addCallback(self.failUnlessIsBarDotTxt)
2073         return d
2074
2075     def test_POST_delete(self):
2076         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2077         d.addCallback(lambda res: self._foo_node.list())
2078         def _check(children):
2079             self.failIf(u"bar.txt" in children)
2080         d.addCallback(_check)
2081         return d
2082
2083     def test_POST_rename_file(self):
2084         d = self.POST(self.public_url + "/foo", t="rename",
2085                       from_name="bar.txt", to_name='wibble.txt')
2086         d.addCallback(lambda res:
2087                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2088         d.addCallback(lambda res:
2089                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2090         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2091         d.addCallback(self.failUnlessIsBarDotTxt)
2092         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2093         d.addCallback(self.failUnlessIsBarJSON)
2094         return d
2095
2096     def test_POST_rename_file_redundant(self):
2097         d = self.POST(self.public_url + "/foo", t="rename",
2098                       from_name="bar.txt", to_name='bar.txt')
2099         d.addCallback(lambda res:
2100                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2101         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2102         d.addCallback(self.failUnlessIsBarDotTxt)
2103         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2104         d.addCallback(self.failUnlessIsBarJSON)
2105         return d
2106
2107     def test_POST_rename_file_replace(self):
2108         # rename a file and replace a directory with it
2109         d = self.POST(self.public_url + "/foo", t="rename",
2110                       from_name="bar.txt", to_name='empty')
2111         d.addCallback(lambda res:
2112                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2113         d.addCallback(lambda res:
2114                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2115         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2116         d.addCallback(self.failUnlessIsBarDotTxt)
2117         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2118         d.addCallback(self.failUnlessIsBarJSON)
2119         return d
2120
2121     def test_POST_rename_file_no_replace_queryarg(self):
2122         # rename a file and replace a directory with it
2123         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2124                       from_name="bar.txt", to_name='empty')
2125         d.addBoth(self.shouldFail, error.Error,
2126                   "POST_rename_file_no_replace_queryarg",
2127                   "409 Conflict",
2128                   "There was already a child by that name, and you asked me "
2129                   "to not replace it")
2130         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2131         d.addCallback(self.failUnlessIsEmptyJSON)
2132         return d
2133
2134     def test_POST_rename_file_no_replace_field(self):
2135         # rename a file and replace a directory with it
2136         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2137                       from_name="bar.txt", to_name='empty')
2138         d.addBoth(self.shouldFail, error.Error,
2139                   "POST_rename_file_no_replace_field",
2140                   "409 Conflict",
2141                   "There was already a child by that name, and you asked me "
2142                   "to not replace it")
2143         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2144         d.addCallback(self.failUnlessIsEmptyJSON)
2145         return d
2146
2147     def failUnlessIsEmptyJSON(self, res):
2148         data = simplejson.loads(res)
2149         self.failUnlessEqual(data[0], "dirnode", data)
2150         self.failUnlessEqual(len(data[1]["children"]), 0)
2151
2152     def test_POST_rename_file_slash_fail(self):
2153         d = self.POST(self.public_url + "/foo", t="rename",
2154                       from_name="bar.txt", to_name='kirk/spock.txt')
2155         d.addBoth(self.shouldFail, error.Error,
2156                   "test_POST_rename_file_slash_fail",
2157                   "400 Bad Request",
2158                   "to_name= may not contain a slash",
2159                   )
2160         d.addCallback(lambda res:
2161                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2162         return d
2163
2164     def test_POST_rename_dir(self):
2165         d = self.POST(self.public_url, t="rename",
2166                       from_name="foo", to_name='plunk')
2167         d.addCallback(lambda res:
2168                       self.failIfNodeHasChild(self.public_root, u"foo"))
2169         d.addCallback(lambda res:
2170                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2171         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2172         d.addCallback(self.failUnlessIsFooJSON)
2173         return d
2174
2175     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2176         """ If target is not None then the redirection has to go to target.  If
2177         statuscode is not None then the redirection has to be accomplished with
2178         that HTTP status code."""
2179         if not isinstance(res, failure.Failure):
2180             to_where = (target is None) and "somewhere" or ("to " + target)
2181             self.fail("%s: we were expecting to get redirected %s, not get an"
2182                       " actual page: %s" % (which, to_where, res))
2183         res.trap(error.PageRedirect)
2184         if statuscode is not None:
2185             self.failUnlessEqual(res.value.status, statuscode,
2186                                  "%s: not a redirect" % which)
2187         if target is not None:
2188             # the PageRedirect does not seem to capture the uri= query arg
2189             # properly, so we can't check for it.
2190             realtarget = self.webish_url + target
2191             self.failUnlessEqual(res.value.location, realtarget,
2192                                  "%s: wrong target" % which)
2193         return res.value.location
2194
2195     def test_GET_URI_form(self):
2196         base = "/uri?uri=%s" % self._bar_txt_uri
2197         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2198         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2199         d = self.GET(base)
2200         d.addBoth(self.shouldRedirect, targetbase)
2201         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2202         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2203         d.addCallback(lambda res: self.GET(base+"&t=json"))
2204         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2205         d.addCallback(self.log, "about to get file by uri")
2206         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2207         d.addCallback(self.failUnlessIsBarDotTxt)
2208         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2209         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2210                                            followRedirect=True))
2211         d.addCallback(self.failUnlessIsFooJSON)
2212         d.addCallback(self.log, "got dir by uri")
2213
2214         return d
2215
2216     def test_GET_URI_form_bad(self):
2217         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2218                              "400 Bad Request", "GET /uri requires uri=",
2219                              self.GET, "/uri")
2220         return d
2221
2222     def test_GET_rename_form(self):
2223         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2224                      followRedirect=True)
2225         def _check(res):
2226             self.failUnless('name="when_done" value="."' in res, res)
2227             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2228         d.addCallback(_check)
2229         return d
2230
2231     def log(self, res, msg):
2232         #print "MSG: %s  RES: %s" % (msg, res)
2233         log.msg(msg)
2234         return res
2235
2236     def test_GET_URI_URL(self):
2237         base = "/uri/%s" % self._bar_txt_uri
2238         d = self.GET(base)
2239         d.addCallback(self.failUnlessIsBarDotTxt)
2240         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2241         d.addCallback(self.failUnlessIsBarDotTxt)
2242         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2243         d.addCallback(self.failUnlessIsBarDotTxt)
2244         return d
2245
2246     def test_GET_URI_URL_dir(self):
2247         base = "/uri/%s?t=json" % self._foo_uri
2248         d = self.GET(base)
2249         d.addCallback(self.failUnlessIsFooJSON)
2250         return d
2251
2252     def test_GET_URI_URL_missing(self):
2253         base = "/uri/%s" % self._bad_file_uri
2254         d = self.GET(base)
2255         d.addBoth(self.shouldHTTPError, "test_GET_URI_URL_missing",
2256                   http.GONE, response_substring="NotEnoughSharesError")
2257         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2258         # here? we must arrange for a download to fail after target.open()
2259         # has been called, and then inspect the response to see that it is
2260         # shorter than we expected.
2261         return d
2262
2263     def test_PUT_DIRURL_uri(self):
2264         d = self.s.create_empty_dirnode()
2265         def _made_dir(dn):
2266             new_uri = dn.get_uri()
2267             # replace /foo with a new (empty) directory
2268             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2269             d.addCallback(lambda res:
2270                           self.failUnlessEqual(res.strip(), new_uri))
2271             d.addCallback(lambda res:
2272                           self.failUnlessChildURIIs(self.public_root,
2273                                                     u"foo",
2274                                                     new_uri))
2275             return d
2276         d.addCallback(_made_dir)
2277         return d
2278
2279     def test_PUT_DIRURL_uri_noreplace(self):
2280         d = self.s.create_empty_dirnode()
2281         def _made_dir(dn):
2282             new_uri = dn.get_uri()
2283             # replace /foo with a new (empty) directory, but ask that
2284             # replace=false, so it should fail
2285             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2286                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2287                                  self.PUT,
2288                                  self.public_url + "/foo?t=uri&replace=false",
2289                                  new_uri)
2290             d.addCallback(lambda res:
2291                           self.failUnlessChildURIIs(self.public_root,
2292                                                     u"foo",
2293                                                     self._foo_uri))
2294             return d
2295         d.addCallback(_made_dir)
2296         return d
2297
2298     def test_PUT_DIRURL_bad_t(self):
2299         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2300                                  "400 Bad Request", "PUT to a directory",
2301                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2302         d.addCallback(lambda res:
2303                       self.failUnlessChildURIIs(self.public_root,
2304                                                 u"foo",
2305                                                 self._foo_uri))
2306         return d
2307
2308     def test_PUT_NEWFILEURL_uri(self):
2309         contents, n, new_uri = self.makefile(8)
2310         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2311         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2312         d.addCallback(lambda res:
2313                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2314                                                       contents))
2315         return d
2316
2317     def test_PUT_NEWFILEURL_uri_replace(self):
2318         contents, n, new_uri = self.makefile(8)
2319         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2320         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2321         d.addCallback(lambda res:
2322                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2323                                                       contents))
2324         return d
2325
2326     def test_PUT_NEWFILEURL_uri_no_replace(self):
2327         contents, n, new_uri = self.makefile(8)
2328         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2329         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2330                   "409 Conflict",
2331                   "There was already a child by that name, and you asked me "
2332                   "to not replace it")
2333         return d
2334
2335     def test_PUT_NEWFILE_URI(self):
2336         file_contents = "New file contents here\n"
2337         d = self.PUT("/uri", file_contents)
2338         def _check(uri):
2339             assert isinstance(uri, str), uri
2340             self.failUnless(uri in FakeCHKFileNode.all_contents)
2341             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2342                                  file_contents)
2343             return self.GET("/uri/%s" % uri)
2344         d.addCallback(_check)
2345         def _check2(res):
2346             self.failUnlessEqual(res, file_contents)
2347         d.addCallback(_check2)
2348         return d
2349
2350     def test_PUT_NEWFILE_URI_only_PUT(self):
2351         d = self.PUT("/uri?t=bogus", "")
2352         d.addBoth(self.shouldFail, error.Error,
2353                   "PUT_NEWFILE_URI_only_PUT",
2354                   "400 Bad Request",
2355                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2356         return d
2357
2358     def test_PUT_NEWFILE_URI_mutable(self):
2359         file_contents = "New file contents here\n"
2360         d = self.PUT("/uri?mutable=true", file_contents)
2361         def _check_mutable(uri):
2362             uri = uri.strip()
2363             u = IURI(uri)
2364             self.failUnless(IMutableFileURI.providedBy(u))
2365             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2366             n = self.s.create_node_from_uri(uri)
2367             return n.download_best_version()
2368         d.addCallback(_check_mutable)
2369         def _check2_mutable(data):
2370             self.failUnlessEqual(data, file_contents)
2371         d.addCallback(_check2_mutable)
2372         return d
2373
2374         def _check(uri):
2375             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2376             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2377                                  file_contents)
2378             return self.GET("/uri/%s" % uri)
2379         d.addCallback(_check)
2380         def _check2(res):
2381             self.failUnlessEqual(res, file_contents)
2382         d.addCallback(_check2)
2383         return d
2384
2385     def test_PUT_mkdir(self):
2386         d = self.PUT("/uri?t=mkdir", "")
2387         def _check(uri):
2388             n = self.s.create_node_from_uri(uri.strip())
2389             d2 = self.failUnlessNodeKeysAre(n, [])
2390             d2.addCallback(lambda res:
2391                            self.GET("/uri/%s?t=json" % uri))
2392             return d2
2393         d.addCallback(_check)
2394         d.addCallback(self.failUnlessIsEmptyJSON)
2395         return d
2396
2397     def test_POST_check(self):
2398         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2399         def _done(res):
2400             # this returns a string form of the results, which are probably
2401             # None since we're using fake filenodes.
2402             # TODO: verify that the check actually happened, by changing
2403             # FakeCHKFileNode to count how many times .check() has been
2404             # called.
2405             pass
2406         d.addCallback(_done)
2407         return d
2408
2409     def test_bad_method(self):
2410         url = self.webish_url + self.public_url + "/foo/bar.txt"
2411         d = self.shouldHTTPError2("test_bad_method",
2412                                   501, "Not Implemented",
2413                                   "I don't know how to treat a BOGUS request.",
2414                                   client.getPage, url, method="BOGUS")
2415         return d
2416
2417     def test_short_url(self):
2418         url = self.webish_url + "/uri"
2419         d = self.shouldHTTPError2("test_short_url", 501, "Not Implemented",
2420                                   "I don't know how to treat a DELETE request.",
2421                                   client.getPage, url, method="DELETE")
2422         return d
2423
2424     def test_ophandle_bad(self):
2425         url = self.webish_url + "/operations/bogus?t=status"
2426         d = self.shouldHTTPError2("test_ophandle_bad", 404, "404 Not Found",
2427                                   "unknown/expired handle 'bogus'",
2428                                   client.getPage, url)
2429         return d
2430
2431     def test_ophandle_cancel(self):
2432         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2433                       followRedirect=True)
2434         d.addCallback(lambda ignored:
2435                       self.GET("/operations/128?t=status&output=JSON"))
2436         def _check1(res):
2437             data = simplejson.loads(res)
2438             self.failUnless("finished" in data, res)
2439             monitor = self.ws.root.child_operations.handles["128"][0]
2440             d = self.POST("/operations/128?t=cancel&output=JSON")
2441             def _check2(res):
2442                 data = simplejson.loads(res)
2443                 self.failUnless("finished" in data, res)
2444                 # t=cancel causes the handle to be forgotten
2445                 self.failUnless(monitor.is_cancelled())
2446             d.addCallback(_check2)
2447             return d
2448         d.addCallback(_check1)
2449         d.addCallback(lambda ignored:
2450                       self.shouldHTTPError2("test_ophandle_cancel",
2451                                             404, "404 Not Found",
2452                                             "unknown/expired handle '128'",
2453                                             self.GET,
2454                                             "/operations/128?t=status&output=JSON"))
2455         return d
2456
2457     def test_ophandle_retainfor(self):
2458         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2459                       followRedirect=True)
2460         d.addCallback(lambda ignored:
2461                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2462         def _check1(res):
2463             data = simplejson.loads(res)
2464             self.failUnless("finished" in data, res)
2465         d.addCallback(_check1)
2466         # the retain-for=0 will cause the handle to be expired very soon
2467         d.addCallback(self.stall, 2.0)
2468         d.addCallback(lambda ignored:
2469                       self.shouldHTTPError2("test_ophandle_retainfor",
2470                                             404, "404 Not Found",
2471                                             "unknown/expired handle '129'",
2472                                             self.GET,
2473                                             "/operations/129?t=status&output=JSON"))
2474         return d
2475
2476     def test_ophandle_release_after_complete(self):
2477         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2478                       followRedirect=True)
2479         d.addCallback(self.wait_for_operation, "130")
2480         d.addCallback(lambda ignored:
2481                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2482         # the release-after-complete=true will cause the handle to be expired
2483         d.addCallback(lambda ignored:
2484                       self.shouldHTTPError2("test_ophandle_release_after_complete",
2485                                             404, "404 Not Found",
2486                                             "unknown/expired handle '130'",
2487                                             self.GET,
2488                                             "/operations/130?t=status&output=JSON"))
2489         return d
2490
2491     def test_incident(self):
2492         d = self.POST("/report_incident", details="eek")
2493         def _done(res):
2494             self.failUnless("Thank you for your report!" in res, res)
2495         d.addCallback(_done)
2496         return d
2497
2498     def test_static(self):
2499         webdir = os.path.join(self.staticdir, "subdir")
2500         fileutil.make_dirs(webdir)
2501         f = open(os.path.join(webdir, "hello.txt"), "wb")
2502         f.write("hello")
2503         f.close()
2504
2505         d = self.GET("/static/subdir/hello.txt")
2506         def _check(res):
2507             self.failUnlessEqual(res, "hello")
2508         d.addCallback(_check)
2509         return d
2510
2511
2512 class Util(unittest.TestCase):
2513     def test_abbreviate_time(self):
2514         self.failUnlessEqual(common.abbreviate_time(None), "")
2515         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2516         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2517         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2518         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2519
2520     def test_abbreviate_rate(self):
2521         self.failUnlessEqual(common.abbreviate_rate(None), "")
2522         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2523         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2524         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2525
2526     def test_abbreviate_size(self):
2527         self.failUnlessEqual(common.abbreviate_size(None), "")
2528         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2529         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2530         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2531         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2532
2533
2534 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase):
2535
2536     def GET(self, urlpath, followRedirect=False, return_response=False,
2537             method="GET", clientnum=0, **kwargs):
2538         # if return_response=True, this fires with (data, statuscode,
2539         # respheaders) instead of just data.
2540         assert not isinstance(urlpath, unicode)
2541         url = self.client_baseurls[clientnum] + urlpath
2542         factory = HTTPClientGETFactory(url, method=method,
2543                                        followRedirect=followRedirect, **kwargs)
2544         reactor.connectTCP("localhost", self.client_webports[clientnum],factory)
2545         d = factory.deferred
2546         def _got_data(data):
2547             return (data, factory.status, factory.response_headers)
2548         if return_response:
2549             d.addCallback(_got_data)
2550         return factory.deferred
2551
2552     def CHECK(self, ign, which, args, clientnum=0):
2553         fileurl = self.fileurls[which]
2554         url = fileurl + "?" + args
2555         return self.GET(url, method="POST", clientnum=clientnum)
2556
2557     def test_filecheck(self):
2558         self.basedir = "web/Grid/filecheck"
2559         self.set_up_grid()
2560         c0 = self.g.clients[0]
2561         self.uris = {}
2562         DATA = "data" * 100
2563         d = c0.upload(upload.Data(DATA, convergence=""))
2564         def _stash_uri(ur, which):
2565             self.uris[which] = ur.uri
2566         d.addCallback(_stash_uri, "good")
2567         d.addCallback(lambda ign:
2568                       c0.upload(upload.Data(DATA+"1", convergence="")))
2569         d.addCallback(_stash_uri, "sick")
2570         d.addCallback(lambda ign:
2571                       c0.upload(upload.Data(DATA+"2", convergence="")))
2572         d.addCallback(_stash_uri, "dead")
2573         def _stash_mutable_uri(n, which):
2574             self.uris[which] = n.get_uri()
2575             assert isinstance(self.uris[which], str)
2576         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2577         d.addCallback(_stash_mutable_uri, "corrupt")
2578         d.addCallback(lambda ign:
2579                       c0.upload(upload.Data("literal", convergence="")))
2580         d.addCallback(_stash_uri, "small")
2581
2582         def _compute_fileurls(ignored):
2583             self.fileurls = {}
2584             for which in self.uris:
2585                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2586         d.addCallback(_compute_fileurls)
2587
2588         def _clobber_shares(ignored):
2589             good_shares = self.find_shares(self.uris["good"])
2590             self.failUnlessEqual(len(good_shares), 10)
2591             sick_shares = self.find_shares(self.uris["sick"])
2592             os.unlink(sick_shares[0][2])
2593             dead_shares = self.find_shares(self.uris["dead"])
2594             for i in range(1, 10):
2595                 os.unlink(dead_shares[i][2])
2596             c_shares = self.find_shares(self.uris["corrupt"])
2597             cso = CorruptShareOptions()
2598             cso.stdout = StringIO()
2599             cso.parseOptions([c_shares[0][2]])
2600             corrupt_share(cso)
2601         d.addCallback(_clobber_shares)
2602
2603         d.addCallback(self.CHECK, "good", "t=check")
2604         def _got_html_good(res):
2605             self.failUnless("Healthy" in res, res)
2606             self.failIf("Not Healthy" in res, res)
2607         d.addCallback(_got_html_good)
2608         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2609         def _got_html_good_return_to(res):
2610             self.failUnless("Healthy" in res, res)
2611             self.failIf("Not Healthy" in res, res)
2612             self.failUnless('<a href="somewhere">Return to parent directory'
2613                             in res, res)
2614         d.addCallback(_got_html_good_return_to)
2615         d.addCallback(self.CHECK, "good", "t=check&output=json")
2616         def _got_json_good(res):
2617             r = simplejson.loads(res)
2618             self.failUnlessEqual(r["summary"], "Healthy")
2619             self.failUnless(r["results"]["healthy"])
2620             self.failIf(r["results"]["needs-rebalancing"])
2621             self.failUnless(r["results"]["recoverable"])
2622         d.addCallback(_got_json_good)
2623
2624         d.addCallback(self.CHECK, "small", "t=check")
2625         def _got_html_small(res):
2626             self.failUnless("Literal files are always healthy" in res, res)
2627             self.failIf("Not Healthy" in res, res)
2628         d.addCallback(_got_html_small)
2629         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2630         def _got_html_small_return_to(res):
2631             self.failUnless("Literal files are always healthy" in res, res)
2632             self.failIf("Not Healthy" in res, res)
2633             self.failUnless('<a href="somewhere">Return to parent directory'
2634                             in res, res)
2635         d.addCallback(_got_html_small_return_to)
2636         d.addCallback(self.CHECK, "small", "t=check&output=json")
2637         def _got_json_small(res):
2638             r = simplejson.loads(res)
2639             self.failUnlessEqual(r["storage-index"], "")
2640             self.failUnless(r["results"]["healthy"])
2641         d.addCallback(_got_json_small)
2642
2643         d.addCallback(self.CHECK, "sick", "t=check")
2644         def _got_html_sick(res):
2645             self.failUnless("Not Healthy" in res, res)
2646         d.addCallback(_got_html_sick)
2647         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2648         def _got_json_sick(res):
2649             r = simplejson.loads(res)
2650             self.failUnlessEqual(r["summary"],
2651                                  "Not Healthy: 9 shares (enc 3-of-10)")
2652             self.failIf(r["results"]["healthy"])
2653             self.failIf(r["results"]["needs-rebalancing"])
2654             self.failUnless(r["results"]["recoverable"])
2655         d.addCallback(_got_json_sick)
2656
2657         d.addCallback(self.CHECK, "dead", "t=check")
2658         def _got_html_dead(res):
2659             self.failUnless("Not Healthy" in res, res)
2660         d.addCallback(_got_html_dead)
2661         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2662         def _got_json_dead(res):
2663             r = simplejson.loads(res)
2664             self.failUnlessEqual(r["summary"],
2665                                  "Not Healthy: 1 shares (enc 3-of-10)")
2666             self.failIf(r["results"]["healthy"])
2667             self.failIf(r["results"]["needs-rebalancing"])
2668             self.failIf(r["results"]["recoverable"])
2669         d.addCallback(_got_json_dead)
2670
2671         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2672         def _got_html_corrupt(res):
2673             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2674         d.addCallback(_got_html_corrupt)
2675         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2676         def _got_json_corrupt(res):
2677             r = simplejson.loads(res)
2678             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2679                             r["summary"])
2680             self.failIf(r["results"]["healthy"])
2681             self.failUnless(r["results"]["recoverable"])
2682             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2683             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2684         d.addCallback(_got_json_corrupt)
2685
2686         d.addErrback(self.explain_web_error)
2687         return d
2688
2689     def test_repair_html(self):
2690         self.basedir = "web/Grid/repair_html"
2691         self.set_up_grid()
2692         c0 = self.g.clients[0]
2693         self.uris = {}
2694         DATA = "data" * 100
2695         d = c0.upload(upload.Data(DATA, convergence=""))
2696         def _stash_uri(ur, which):
2697             self.uris[which] = ur.uri
2698         d.addCallback(_stash_uri, "good")
2699         d.addCallback(lambda ign:
2700                       c0.upload(upload.Data(DATA+"1", convergence="")))
2701         d.addCallback(_stash_uri, "sick")
2702         d.addCallback(lambda ign:
2703                       c0.upload(upload.Data(DATA+"2", convergence="")))
2704         d.addCallback(_stash_uri, "dead")
2705         def _stash_mutable_uri(n, which):
2706             self.uris[which] = n.get_uri()
2707             assert isinstance(self.uris[which], str)
2708         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2709         d.addCallback(_stash_mutable_uri, "corrupt")
2710
2711         def _compute_fileurls(ignored):
2712             self.fileurls = {}
2713             for which in self.uris:
2714                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2715         d.addCallback(_compute_fileurls)
2716
2717         def _clobber_shares(ignored):
2718             good_shares = self.find_shares(self.uris["good"])
2719             self.failUnlessEqual(len(good_shares), 10)
2720             sick_shares = self.find_shares(self.uris["sick"])
2721             os.unlink(sick_shares[0][2])
2722             dead_shares = self.find_shares(self.uris["dead"])
2723             for i in range(1, 10):
2724                 os.unlink(dead_shares[i][2])
2725             c_shares = self.find_shares(self.uris["corrupt"])
2726             cso = CorruptShareOptions()
2727             cso.stdout = StringIO()
2728             cso.parseOptions([c_shares[0][2]])
2729             corrupt_share(cso)
2730         d.addCallback(_clobber_shares)
2731
2732         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2733         def _got_html_good(res):
2734             self.failUnless("Healthy" in res, res)
2735             self.failIf("Not Healthy" in res, res)
2736             self.failUnless("No repair necessary" in res, res)
2737         d.addCallback(_got_html_good)
2738
2739         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2740         def _got_html_sick(res):
2741             self.failUnless("Healthy : healthy" in res, res)
2742             self.failIf("Not Healthy" in res, res)
2743             self.failUnless("Repair successful" in res, res)
2744         d.addCallback(_got_html_sick)
2745
2746         # repair of a dead file will fail, of course, but it isn't yet
2747         # clear how this should be reported. Right now it shows up as
2748         # a "410 Gone".
2749         #
2750         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2751         #def _got_html_dead(res):
2752         #    print res
2753         #    self.failUnless("Healthy : healthy" in res, res)
2754         #    self.failIf("Not Healthy" in res, res)
2755         #    self.failUnless("No repair necessary" in res, res)
2756         #d.addCallback(_got_html_dead)
2757
2758         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2759         def _got_html_corrupt(res):
2760             self.failUnless("Healthy : Healthy" in res, res)
2761             self.failIf("Not Healthy" in res, res)
2762             self.failUnless("Repair successful" in res, res)
2763         d.addCallback(_got_html_corrupt)
2764
2765         d.addErrback(self.explain_web_error)
2766         return d
2767
2768     def test_repair_json(self):
2769         self.basedir = "web/Grid/repair_json"
2770         self.set_up_grid()
2771         c0 = self.g.clients[0]
2772         self.uris = {}
2773         DATA = "data" * 100
2774         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2775         def _stash_uri(ur, which):
2776             self.uris[which] = ur.uri
2777         d.addCallback(_stash_uri, "sick")
2778
2779         def _compute_fileurls(ignored):
2780             self.fileurls = {}
2781             for which in self.uris:
2782                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2783         d.addCallback(_compute_fileurls)
2784
2785         def _clobber_shares(ignored):
2786             sick_shares = self.find_shares(self.uris["sick"])
2787             os.unlink(sick_shares[0][2])
2788         d.addCallback(_clobber_shares)
2789
2790         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2791         def _got_json_sick(res):
2792             r = simplejson.loads(res)
2793             self.failUnlessEqual(r["repair-attempted"], True)
2794             self.failUnlessEqual(r["repair-successful"], True)
2795             self.failUnlessEqual(r["pre-repair-results"]["summary"],
2796                                  "Not Healthy: 9 shares (enc 3-of-10)")
2797             self.failIf(r["pre-repair-results"]["results"]["healthy"])
2798             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2799             self.failUnless(r["post-repair-results"]["results"]["healthy"])
2800         d.addCallback(_got_json_sick)
2801
2802         d.addErrback(self.explain_web_error)
2803         return d
2804
2805     def test_deep_check(self):
2806         self.basedir = "web/Grid/deep_check"
2807         self.set_up_grid()
2808         c0 = self.g.clients[0]
2809         self.uris = {}
2810         self.fileurls = {}
2811         DATA = "data" * 100
2812         d = c0.create_empty_dirnode()
2813         def _stash_root_and_create_file(n):
2814             self.rootnode = n
2815             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2816             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2817         d.addCallback(_stash_root_and_create_file)
2818         def _stash_uri(fn, which):
2819             self.uris[which] = fn.get_uri()
2820         d.addCallback(_stash_uri, "good")
2821         d.addCallback(lambda ign:
2822                       self.rootnode.add_file(u"small",
2823                                              upload.Data("literal",
2824                                                         convergence="")))
2825         d.addCallback(_stash_uri, "small")
2826
2827         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2828         def _done(res):
2829             units = [simplejson.loads(line)
2830                      for line in res.splitlines()
2831                      if line]
2832             self.failUnlessEqual(len(units), 3+1)
2833             # should be parent-first
2834             u0 = units[0]
2835             self.failUnlessEqual(u0["path"], [])
2836             self.failUnlessEqual(u0["type"], "directory")
2837             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2838             u0cr = u0["check-results"]
2839             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2840
2841             ugood = [u for u in units
2842                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2843             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2844             ugoodcr = ugood["check-results"]
2845             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2846
2847             stats = units[-1]
2848             self.failUnlessEqual(stats["type"], "stats")
2849             s = stats["stats"]
2850             self.failUnlessEqual(s["count-immutable-files"], 1)
2851             self.failUnlessEqual(s["count-literal-files"], 1)
2852             self.failUnlessEqual(s["count-directories"], 1)
2853         d.addCallback(_done)
2854
2855         d.addErrback(self.explain_web_error)
2856         return d
2857
2858     def test_deep_check_and_repair(self):
2859         self.basedir = "web/Grid/deep_check_and_repair"
2860         self.set_up_grid()
2861         c0 = self.g.clients[0]
2862         self.uris = {}
2863         self.fileurls = {}
2864         DATA = "data" * 100
2865         d = c0.create_empty_dirnode()
2866         def _stash_root_and_create_file(n):
2867             self.rootnode = n
2868             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2869             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2870         d.addCallback(_stash_root_and_create_file)
2871         def _stash_uri(fn, which):
2872             self.uris[which] = fn.get_uri()
2873         d.addCallback(_stash_uri, "good")
2874         d.addCallback(lambda ign:
2875                       self.rootnode.add_file(u"small",
2876                                              upload.Data("literal",
2877                                                         convergence="")))
2878         d.addCallback(_stash_uri, "small")
2879         d.addCallback(lambda ign:
2880                       self.rootnode.add_file(u"sick",
2881                                              upload.Data(DATA+"1",
2882                                                         convergence="")))
2883         d.addCallback(_stash_uri, "sick")
2884         #d.addCallback(lambda ign:
2885         #              self.rootnode.add_file(u"dead",
2886         #                                     upload.Data(DATA+"2",
2887         #                                                convergence="")))
2888         #d.addCallback(_stash_uri, "dead")
2889
2890         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2891         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2892         #d.addCallback(_stash_uri, "corrupt")
2893
2894         def _clobber_shares(ignored):
2895             good_shares = self.find_shares(self.uris["good"])
2896             self.failUnlessEqual(len(good_shares), 10)
2897             sick_shares = self.find_shares(self.uris["sick"])
2898             os.unlink(sick_shares[0][2])
2899             #dead_shares = self.find_shares(self.uris["dead"])
2900             #for i in range(1, 10):
2901             #    os.unlink(dead_shares[i][2])
2902
2903             #c_shares = self.find_shares(self.uris["corrupt"])
2904             #cso = CorruptShareOptions()
2905             #cso.stdout = StringIO()
2906             #cso.parseOptions([c_shares[0][2]])
2907             #corrupt_share(cso)
2908         d.addCallback(_clobber_shares)
2909
2910         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2911         def _done(res):
2912             units = [simplejson.loads(line)
2913                      for line in res.splitlines()
2914                      if line]
2915             self.failUnlessEqual(len(units), 4+1)
2916             # should be parent-first
2917             u0 = units[0]
2918             self.failUnlessEqual(u0["path"], [])
2919             self.failUnlessEqual(u0["type"], "directory")
2920             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2921             u0crr = u0["check-and-repair-results"]
2922             self.failUnlessEqual(u0crr["repair-attempted"], False)
2923             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2924
2925             ugood = [u for u in units
2926                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2927             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2928             ugoodcrr = ugood["check-and-repair-results"]
2929             self.failUnlessEqual(u0crr["repair-attempted"], False)
2930             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2931
2932             usick = [u for u in units
2933                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
2934             self.failUnlessEqual(usick["cap"], self.uris["sick"])
2935             usickcrr = usick["check-and-repair-results"]
2936             self.failUnlessEqual(usickcrr["repair-attempted"], True)
2937             self.failUnlessEqual(usickcrr["repair-successful"], True)
2938             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2939             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2940
2941             stats = units[-1]
2942             self.failUnlessEqual(stats["type"], "stats")
2943             s = stats["stats"]
2944             self.failUnlessEqual(s["count-immutable-files"], 2)
2945             self.failUnlessEqual(s["count-literal-files"], 1)
2946             self.failUnlessEqual(s["count-directories"], 1)
2947         d.addCallback(_done)
2948
2949         d.addErrback(self.explain_web_error)
2950         return d
2951
2952     def _count_leases(self, ignored, which):
2953         u = self.uris[which]
2954         shares = self.find_shares(u)
2955         lease_counts = []
2956         for shnum, serverid, fn in shares:
2957             if u.startswith("URI:SSK") or u.startswith("URI:DIR2"):
2958                 sf = MutableShareFile(fn)
2959                 num_leases = len(sf.debug_get_leases())
2960             elif u.startswith("URI:CHK"):
2961                 sf = ShareFile(fn)
2962                 num_leases = len(list(sf.iter_leases()))
2963             else:
2964                 raise RuntimeError("can't get leases on %s" % u)
2965         lease_counts.append( (fn, num_leases) )
2966         return lease_counts
2967
2968     def _assert_leasecount(self, lease_counts, expected):
2969         for (fn, num_leases) in lease_counts:
2970             if num_leases != expected:
2971                 self.fail("expected %d leases, have %d, on %s" %
2972                           (expected, num_leases, fn))
2973
2974     def test_add_lease(self):
2975         self.basedir = "web/Grid/add_lease"
2976         self.set_up_grid(num_clients=2)
2977         c0 = self.g.clients[0]
2978         self.uris = {}
2979         DATA = "data" * 100
2980         d = c0.upload(upload.Data(DATA, convergence=""))
2981         def _stash_uri(ur, which):
2982             self.uris[which] = ur.uri
2983         d.addCallback(_stash_uri, "one")
2984         d.addCallback(lambda ign:
2985                       c0.upload(upload.Data(DATA+"1", convergence="")))
2986         d.addCallback(_stash_uri, "two")
2987         def _stash_mutable_uri(n, which):
2988             self.uris[which] = n.get_uri()
2989             assert isinstance(self.uris[which], str)
2990         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
2991         d.addCallback(_stash_mutable_uri, "mutable")
2992
2993         def _compute_fileurls(ignored):
2994             self.fileurls = {}
2995             for which in self.uris:
2996                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2997         d.addCallback(_compute_fileurls)
2998
2999         d.addCallback(self._count_leases, "one")
3000         d.addCallback(self._assert_leasecount, 1)
3001         d.addCallback(self._count_leases, "two")
3002         d.addCallback(self._assert_leasecount, 1)
3003         d.addCallback(self._count_leases, "mutable")
3004         d.addCallback(self._assert_leasecount, 1)
3005
3006         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3007         def _got_html_good(res):
3008             self.failUnless("Healthy" in res, res)
3009             self.failIf("Not Healthy" in res, res)
3010         d.addCallback(_got_html_good)
3011
3012         d.addCallback(self._count_leases, "one")
3013         d.addCallback(self._assert_leasecount, 1)
3014         d.addCallback(self._count_leases, "two")
3015         d.addCallback(self._assert_leasecount, 1)
3016         d.addCallback(self._count_leases, "mutable")
3017         d.addCallback(self._assert_leasecount, 1)
3018
3019         # this CHECK uses the original client, which uses the same
3020         # lease-secrets, so it will just renew the original lease
3021         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3022         d.addCallback(_got_html_good)
3023
3024         d.addCallback(self._count_leases, "one")
3025         d.addCallback(self._assert_leasecount, 1)
3026         d.addCallback(self._count_leases, "two")
3027         d.addCallback(self._assert_leasecount, 1)
3028         d.addCallback(self._count_leases, "mutable")
3029         d.addCallback(self._assert_leasecount, 1)
3030
3031         # this CHECK uses an alternate client, which adds a second lease
3032         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3033         d.addCallback(_got_html_good)
3034
3035         d.addCallback(self._count_leases, "one")
3036         d.addCallback(self._assert_leasecount, 2)
3037         d.addCallback(self._count_leases, "two")
3038         d.addCallback(self._assert_leasecount, 1)
3039         d.addCallback(self._count_leases, "mutable")
3040         d.addCallback(self._assert_leasecount, 1)
3041
3042         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3043         d.addCallback(_got_html_good)
3044
3045         d.addCallback(self._count_leases, "one")
3046         d.addCallback(self._assert_leasecount, 2)
3047         d.addCallback(self._count_leases, "two")
3048         d.addCallback(self._assert_leasecount, 1)
3049         d.addCallback(self._count_leases, "mutable")
3050         d.addCallback(self._assert_leasecount, 1)
3051
3052         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3053                       clientnum=1)
3054         d.addCallback(_got_html_good)
3055
3056         d.addCallback(self._count_leases, "one")
3057         d.addCallback(self._assert_leasecount, 2)
3058         d.addCallback(self._count_leases, "two")
3059         d.addCallback(self._assert_leasecount, 1)
3060         d.addCallback(self._count_leases, "mutable")
3061         d.addCallback(self._assert_leasecount, 2)
3062
3063         d.addErrback(self.explain_web_error)
3064         return d
3065
3066     def test_deep_add_lease(self):
3067         self.basedir = "web/Grid/deep_add_lease"
3068         self.set_up_grid(num_clients=2)
3069         c0 = self.g.clients[0]
3070         self.uris = {}
3071         self.fileurls = {}
3072         DATA = "data" * 100
3073         d = c0.create_empty_dirnode()
3074         def _stash_root_and_create_file(n):
3075             self.rootnode = n
3076             self.uris["root"] = n.get_uri()
3077             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3078             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3079         d.addCallback(_stash_root_and_create_file)
3080         def _stash_uri(fn, which):
3081             self.uris[which] = fn.get_uri()
3082         d.addCallback(_stash_uri, "one")
3083         d.addCallback(lambda ign:
3084                       self.rootnode.add_file(u"small",
3085                                              upload.Data("literal",
3086                                                         convergence="")))
3087         d.addCallback(_stash_uri, "small")
3088
3089         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3090         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3091         d.addCallback(_stash_uri, "mutable")
3092
3093         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3094         def _done(res):
3095             units = [simplejson.loads(line)
3096                      for line in res.splitlines()
3097                      if line]
3098             # root, one, small, mutable,   stats
3099             self.failUnlessEqual(len(units), 4+1)
3100         d.addCallback(_done)
3101
3102         d.addCallback(self._count_leases, "root")
3103         d.addCallback(self._assert_leasecount, 1)
3104         d.addCallback(self._count_leases, "one")
3105         d.addCallback(self._assert_leasecount, 1)
3106         d.addCallback(self._count_leases, "mutable")
3107         d.addCallback(self._assert_leasecount, 1)
3108
3109         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3110         d.addCallback(_done)
3111
3112         d.addCallback(self._count_leases, "root")
3113         d.addCallback(self._assert_leasecount, 1)
3114         d.addCallback(self._count_leases, "one")
3115         d.addCallback(self._assert_leasecount, 1)
3116         d.addCallback(self._count_leases, "mutable")
3117         d.addCallback(self._assert_leasecount, 1)
3118
3119         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3120                       clientnum=1)
3121         d.addCallback(_done)
3122
3123         d.addCallback(self._count_leases, "root")
3124         d.addCallback(self._assert_leasecount, 2)
3125         d.addCallback(self._count_leases, "one")
3126         d.addCallback(self._assert_leasecount, 2)
3127         d.addCallback(self._count_leases, "mutable")
3128         d.addCallback(self._assert_leasecount, 2)
3129
3130         d.addErrback(self.explain_web_error)
3131         return d