src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.immutable import upload, download
  13 from allmydata.web import status, common
  14 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  15 from allmydata.util import fileutil, base32
  16 from allmydata.util.assertutil import precondition
  17 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  18      FakeMutableFileNode, create_chk_filenode, WebErrorMixin, ShouldFailMixin
  19 from allmydata.interfaces import IURI, INewDirectoryURI, \
  20      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  21 from allmydata.mutable import servermap, publish, retrieve
  22 import common_util as testutil
  23 from allmydata.test.no_network import GridTestMixin
  24
  25 from allmydata.test.common_web import HTTPClientGETFactory, \
  26      HTTPClientHEADFactory
  27
  28 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  29 # create a webserver that works against them
  30
  31 class FakeIntroducerClient:
  32     def get_all_connectors(self):
  33         return {}
  34     def get_all_connections_for(self, service_name):
  35         return frozenset()
  36     def get_all_peerids(self):
  37         return frozenset()
  38
  39 class FakeStatsProvider:
  40     def get_stats(self):
  41         stats = {'stats': {}, 'counters': {}}
  42         return stats
  43
  44 class FakeClient(service.MultiService):
  45     nodeid = "fake_nodeid"
  46     nickname = "fake_nickname"
  47     basedir = "fake_basedir"
  48     def get_versions(self):
  49         return {'allmydata': "fake",
  50                 'foolscap': "fake",
  51                 'twisted': "fake",
  52                 'zfec': "fake",
  53                 }
  54     introducer_furl = "None"
  55     introducer_client = FakeIntroducerClient()
  56     _all_upload_status = [upload.UploadStatus()]
  57     _all_download_status = [download.DownloadStatus()]
  58     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  59     _all_publish_statuses = [publish.PublishStatus()]
  60     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  61     convergence = "some random string"
  62     stats_provider = FakeStatsProvider()
  63
  64     def connected_to_introducer(self):
  65         return False
  66
  67     def get_nickname_for_peerid(self, peerid):
  68         return u"John Doe"
  69
  70     def get_permuted_peers(self, service_name, key):
  71         return []
  72
  73     def create_node_from_uri(self, auri):
  74         precondition(isinstance(auri, str), auri)
  75         u = uri.from_string(auri)
  76         if (INewDirectoryURI.providedBy(u)
  77             or IReadonlyNewDirectoryURI.providedBy(u)):
  78             return FakeDirectoryNode(self).init_from_uri(u)
  79         if IFileURI.providedBy(u):
  80             return FakeCHKFileNode(u, self)
  81         assert IMutableFileURI.providedBy(u), u
  82         return FakeMutableFileNode(self).init_from_uri(u)
  83
  84     def create_empty_dirnode(self):
  85         n = FakeDirectoryNode(self)
  86         d = n.create()
  87         d.addCallback(lambda res: n)
  88         return d
  89
  90     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  91     def create_mutable_file(self, contents=""):
  92         n = FakeMutableFileNode(self)
  93         return n.create(contents)
  94
  95     def upload(self, uploadable):
  96         d = uploadable.get_size()
  97         d.addCallback(lambda size: uploadable.read(size))
  98         def _got_data(datav):
  99             data = "".join(datav)
 100             n = create_chk_filenode(self, data)
 101             results = upload.UploadResults()
 102             results.uri = n.get_uri()
 103             return results
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def list_all_upload_statuses(self):
 108         return self._all_upload_status
 109     def list_all_download_statuses(self):
 110         return self._all_download_status
 111     def list_all_mapupdate_statuses(self):
 112         return self._all_mapupdate_statuses
 113     def list_all_publish_statuses(self):
 114         return self._all_publish_statuses
 115     def list_all_retrieve_statuses(self):
 116         return self._all_retrieve_statuses
 117     def list_all_helper_statuses(self):
 118         return []
 119
 120 class WebMixin(object):
 121     def setUp(self):
 122         self.s = FakeClient()
 123         self.s.startService()
 124         self.staticdir = self.mktemp()
 125         self.ws = s = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 126         s.setServiceParent(self.s)
 127         self.webish_port = port = s.listener._port.getHost().port
 128         self.webish_url = "http://localhost:%d" % port
 129
 130         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 131         d = defer.DeferredList(l)
 132         def _then(res):
 133             self.public_root = res[0][1]
 134             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 135             self.public_url = "/uri/" + self.public_root.get_uri()
 136             self.private_root = res[1][1]
 137
 138             foo = res[2][1]
 139             self._foo_node = foo
 140             self._foo_uri = foo.get_uri()
 141             self._foo_readonly_uri = foo.get_readonly_uri()
 142             self._foo_verifycap = foo.get_verify_cap().to_string()
 143             # NOTE: we ignore the deferred on all set_uri() calls, because we
 144             # know the fake nodes do these synchronously
 145             self.public_root.set_uri(u"foo", foo.get_uri())
 146
 147             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 148             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 149             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 150
 151             foo.set_uri(u"empty", res[3][1].get_uri())
 152             sub_uri = res[4][1].get_uri()
 153             self._sub_uri = sub_uri
 154             foo.set_uri(u"sub", sub_uri)
 155             sub = self.s.create_node_from_uri(sub_uri)
 156
 157             _ign, n, blocking_uri = self.makefile(1)
 158             foo.set_uri(u"blockingfile", blocking_uri)
 159
 160             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 161             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 162             # still think of it as an umlaut
 163             foo.set_uri(unicode_filename, self._bar_txt_uri)
 164
 165             _ign, n, baz_file = self.makefile(2)
 166             self._baz_file_uri = baz_file
 167             sub.set_uri(u"baz.txt", baz_file)
 168
 169             _ign, n, self._bad_file_uri = self.makefile(3)
 170             # this uri should not be downloadable
 171             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 172
 173             rodir = res[5][1]
 174             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 175             rodir.set_uri(u"nor", baz_file)
 176
 177             # public/
 178             # public/foo/
 179             # public/foo/bar.txt
 180             # public/foo/blockingfile
 181             # public/foo/empty/
 182             # public/foo/sub/
 183             # public/foo/sub/baz.txt
 184             # public/reedownlee/
 185             # public/reedownlee/nor
 186             self.NEWFILE_CONTENTS = "newfile contents\n"
 187
 188             return foo.get_metadata_for(u"bar.txt")
 189         d.addCallback(_then)
 190         def _got_metadata(metadata):
 191             self._bar_txt_metadata = metadata
 192         d.addCallback(_got_metadata)
 193         return d
 194
 195     def makefile(self, number):
 196         contents = "contents of file %s\n" % number
 197         n = create_chk_filenode(self.s, contents)
 198         return contents, n, n.get_uri()
 199
 200     def tearDown(self):
 201         return self.s.stopService()
 202
 203     def failUnlessIsBarDotTxt(self, res):
 204         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 205
 206     def failUnlessIsBarJSON(self, res):
 207         data = simplejson.loads(res)
 208         self.failUnless(isinstance(data, list))
 209         self.failUnlessEqual(data[0], u"filenode")
 210         self.failUnless(isinstance(data[1], dict))
 211         self.failIf(data[1]["mutable"])
 212         self.failIf("rw_uri" in data[1]) # immutable
 213         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 214         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 215         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 216
 217     def failUnlessIsFooJSON(self, res):
 218         data = simplejson.loads(res)
 219         self.failUnless(isinstance(data, list))
 220         self.failUnlessEqual(data[0], "dirnode", res)
 221         self.failUnless(isinstance(data[1], dict))
 222         self.failUnless(data[1]["mutable"])
 223         self.failUnless("rw_uri" in data[1]) # mutable
 224         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 225         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 226         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 227
 228         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 229         self.failUnlessEqual(kidnames,
 230                              [u"bar.txt", u"blockingfile", u"empty",
 231                               u"n\u00fc.txt", u"sub"])
 232         kids = dict( [(unicode(name),value)
 233                       for (name,value)
 234                       in data[1]["children"].iteritems()] )
 235         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 236         self.failUnless("metadata" in kids[u"sub"][1])
 237         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 238         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 239         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 242         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 243                              self._bar_txt_verifycap)
 244         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 245                              self._bar_txt_metadata["ctime"])
 246         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 247                              self._bar_txt_uri)
 248
 249     def GET(self, urlpath, followRedirect=False, return_response=False,
 250             **kwargs):
 251         # if return_response=True, this fires with (data, statuscode,
 252         # respheaders) instead of just data.
 253         assert not isinstance(urlpath, unicode)
 254         url = self.webish_url + urlpath
 255         factory = HTTPClientGETFactory(url, method="GET",
 256                                        followRedirect=followRedirect, **kwargs)
 257         reactor.connectTCP("localhost", self.webish_port, factory)
 258         d = factory.deferred
 259         def _got_data(data):
 260             return (data, factory.status, factory.response_headers)
 261         if return_response:
 262             d.addCallback(_got_data)
 263         return factory.deferred
 264
 265     def HEAD(self, urlpath, return_response=False, **kwargs):
 266         # this requires some surgery, because twisted.web.client doesn't want
 267         # to give us back the response headers.
 268         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 269         reactor.connectTCP("localhost", self.webish_port, factory)
 270         d = factory.deferred
 271         def _got_data(data):
 272             return (data, factory.status, factory.response_headers)
 273         if return_response:
 274             d.addCallback(_got_data)
 275         return factory.deferred
 276
 277     def PUT(self, urlpath, data, **kwargs):
 278         url = self.webish_url + urlpath
 279         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 280
 281     def DELETE(self, urlpath):
 282         url = self.webish_url + urlpath
 283         return client.getPage(url, method="DELETE")
 284
 285     def POST(self, urlpath, followRedirect=False, **fields):
 286         url = self.webish_url + urlpath
 287         sepbase = "boogabooga"
 288         sep = "--" + sepbase
 289         form = []
 290         form.append(sep)
 291         form.append('Content-Disposition: form-data; name="_charset"')
 292         form.append('')
 293         form.append('UTF-8')
 294         form.append(sep)
 295         for name, value in fields.iteritems():
 296             if isinstance(value, tuple):
 297                 filename, value = value
 298                 form.append('Content-Disposition: form-data; name="%s"; '
 299                             'filename="%s"' % (name, filename.encode("utf-8")))
 300             else:
 301                 form.append('Content-Disposition: form-data; name="%s"' % name)
 302             form.append('')
 303             if isinstance(value, unicode):
 304                 value = value.encode("utf-8")
 305             else:
 306                 value = str(value)
 307             assert isinstance(value, str)
 308             form.append(value)
 309             form.append(sep)
 310         form[-1] += "--"
 311         body = "\r\n".join(form) + "\r\n"
 312         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 313                    }
 314         return client.getPage(url, method="POST", postdata=body,
 315                               headers=headers, followRedirect=followRedirect)
 316
 317     def shouldFail(self, res, expected_failure, which,
 318                    substring=None, response_substring=None):
 319         if isinstance(res, failure.Failure):
 320             res.trap(expected_failure)
 321             if substring:
 322                 self.failUnless(substring in str(res),
 323                                 "substring '%s' not in '%s'"
 324                                 % (substring, str(res)))
 325             if response_substring:
 326                 self.failUnless(response_substring in res.value.response,
 327                                 "response substring '%s' not in '%s'"
 328                                 % (response_substring, res.value.response))
 329         else:
 330             self.fail("%s was supposed to raise %s, not get '%s'" %
 331                       (which, expected_failure, res))
 332
 333     def shouldFail2(self, expected_failure, which, substring,
 334                     response_substring,
 335                     callable, *args, **kwargs):
 336         assert substring is None or isinstance(substring, str)
 337         assert response_substring is None or isinstance(response_substring, str)
 338         d = defer.maybeDeferred(callable, *args, **kwargs)
 339         def done(res):
 340             if isinstance(res, failure.Failure):
 341                 res.trap(expected_failure)
 342                 if substring:
 343                     self.failUnless(substring in str(res),
 344                                     "%s: substring '%s' not in '%s'"
 345                                     % (which, substring, str(res)))
 346                 if response_substring:
 347                     self.failUnless(response_substring in res.value.response,
 348                                     "%s: response substring '%s' not in '%s'"
 349                                     % (which,
 350                                        response_substring, res.value.response))
 351             else:
 352                 self.fail("%s was supposed to raise %s, not get '%s'" %
 353                           (which, expected_failure, res))
 354         d.addBoth(done)
 355         return d
 356
 357     def should404(self, res, which):
 358         if isinstance(res, failure.Failure):
 359             res.trap(error.Error)
 360             self.failUnlessEqual(res.value.status, "404")
 361         else:
 362             self.fail("%s was supposed to Error(404), not get '%s'" %
 363                       (which, res))
 364
 365
 366 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 367     def test_create(self):
 368         pass
 369
 370     def test_welcome(self):
 371         d = self.GET("/")
 372         def _check(res):
 373             self.failUnless('Welcome To AllMyData' in res)
 374             self.failUnless('Tahoe' in res)
 375
 376             self.s.basedir = 'web/test_welcome'
 377             fileutil.make_dirs("web/test_welcome")
 378             fileutil.make_dirs("web/test_welcome/private")
 379             return self.GET("/")
 380         d.addCallback(_check)
 381         return d
 382
 383     def test_provisioning(self):
 384         d = self.GET("/provisioning/")
 385         def _check(res):
 386             self.failUnless('Tahoe Provisioning Tool' in res)
 387             fields = {'filled': True,
 388                       "num_users": int(50e3),
 389                       "files_per_user": 1000,
 390                       "space_per_user": int(1e9),
 391                       "sharing_ratio": 1.0,
 392                       "encoding_parameters": "3-of-10-5",
 393                       "num_servers": 30,
 394                       "ownership_mode": "A",
 395                       "download_rate": 100,
 396                       "upload_rate": 10,
 397                       "delete_rate": 10,
 398                       "lease_timer": 7,
 399                       }
 400             return self.POST("/provisioning/", **fields)
 401
 402         d.addCallback(_check)
 403         def _check2(res):
 404             self.failUnless('Tahoe Provisioning Tool' in res)
 405             self.failUnless("Share space consumed: 167.01TB" in res)
 406
 407             fields = {'filled': True,
 408                       "num_users": int(50e6),
 409                       "files_per_user": 1000,
 410                       "space_per_user": int(5e9),
 411                       "sharing_ratio": 1.0,
 412                       "encoding_parameters": "25-of-100-50",
 413                       "num_servers": 30000,
 414                       "ownership_mode": "E",
 415                       "drive_failure_model": "U",
 416                       "drive_size": 1000,
 417                       "download_rate": 1000,
 418                       "upload_rate": 100,
 419                       "delete_rate": 100,
 420                       "lease_timer": 7,
 421                       }
 422             return self.POST("/provisioning/", **fields)
 423         d.addCallback(_check2)
 424         def _check3(res):
 425             self.failUnless("Share space consumed: huge!" in res)
 426             fields = {'filled': True}
 427             return self.POST("/provisioning/", **fields)
 428         d.addCallback(_check3)
 429         def _check4(res):
 430             self.failUnless("Share space consumed:" in res)
 431         d.addCallback(_check4)
 432         return d
 433
 434     def test_reliability_tool(self):
 435         try:
 436             from allmydata import reliability
 437             _hush_pyflakes = reliability
 438         except:
 439             raise unittest.SkipTest("reliability tool requires NumPy")
 440
 441         d = self.GET("/reliability/")
 442         def _check(res):
 443             self.failUnless('Tahoe Reliability Tool' in res)
 444             fields = {'drive_lifetime': "8Y",
 445                       "k": "3",
 446                       "R": "7",
 447                       "N": "10",
 448                       "delta": "100000",
 449                       "check_period": "1M",
 450                       "report_period": "3M",
 451                       "report_span": "5Y",
 452                       }
 453             return self.POST("/reliability/", **fields)
 454
 455         d.addCallback(_check)
 456         def _check2(res):
 457             self.failUnless('Tahoe Reliability Tool' in res)
 458             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 459             self.failUnless(re.search(r, res), res)
 460         d.addCallback(_check2)
 461         return d
 462
 463     def test_status(self):
 464         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 465         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 466         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 467         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 468         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 469         d = self.GET("/status", followRedirect=True)
 470         def _check(res):
 471             self.failUnless('Upload and Download Status' in res, res)
 472             self.failUnless('"down-%d"' % dl_num in res, res)
 473             self.failUnless('"up-%d"' % ul_num in res, res)
 474             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 475             self.failUnless('"publish-%d"' % pub_num in res, res)
 476             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 477         d.addCallback(_check)
 478         d.addCallback(lambda res: self.GET("/status/?t=json"))
 479         def _check_json(res):
 480             data = simplejson.loads(res)
 481             self.failUnless(isinstance(data, dict))
 482             active = data["active"]
 483             # TODO: test more. We need a way to fake an active operation
 484             # here.
 485         d.addCallback(_check_json)
 486
 487         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 488         def _check_dl(res):
 489             self.failUnless("File Download Status" in res, res)
 490         d.addCallback(_check_dl)
 491         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 492         def _check_ul(res):
 493             self.failUnless("File Upload Status" in res, res)
 494         d.addCallback(_check_ul)
 495         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 496         def _check_mapupdate(res):
 497             self.failUnless("Mutable File Servermap Update Status" in res, res)
 498         d.addCallback(_check_mapupdate)
 499         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 500         def _check_publish(res):
 501             self.failUnless("Mutable File Publish Status" in res, res)
 502         d.addCallback(_check_publish)
 503         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 504         def _check_retrieve(res):
 505             self.failUnless("Mutable File Retrieve Status" in res, res)
 506         d.addCallback(_check_retrieve)
 507
 508         return d
 509
 510     def test_status_numbers(self):
 511         drrm = status.DownloadResultsRendererMixin()
 512         self.failUnlessEqual(drrm.render_time(None, None), "")
 513         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 514         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 515         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 517         self.failUnlessEqual(drrm.render_rate(None, None), "")
 518         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 519         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 521
 522         urrm = status.UploadResultsRendererMixin()
 523         self.failUnlessEqual(urrm.render_time(None, None), "")
 524         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 525         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 526         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 528         self.failUnlessEqual(urrm.render_rate(None, None), "")
 529         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 530         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 532
 533     def test_GET_FILEURL(self):
 534         d = self.GET(self.public_url + "/foo/bar.txt")
 535         d.addCallback(self.failUnlessIsBarDotTxt)
 536         return d
 537
 538     def test_GET_FILEURL_range(self):
 539         headers = {"range": "bytes=1-10"}
 540         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 541                      return_response=True)
 542         def _got((res, status, headers)):
 543             self.failUnlessEqual(int(status), 206)
 544             self.failUnless(headers.has_key("content-range"))
 545             self.failUnlessEqual(headers["content-range"][0],
 546                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 547             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 548         d.addCallback(_got)
 549         return d
 550
 551     def test_GET_FILEURL_partial_range(self):
 552         headers = {"range": "bytes=5-"}
 553         length  = len(self.BAR_CONTENTS)
 554         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 555                      return_response=True)
 556         def _got((res, status, headers)):
 557             self.failUnlessEqual(int(status), 206)
 558             self.failUnless(headers.has_key("content-range"))
 559             self.failUnlessEqual(headers["content-range"][0],
 560                                  "bytes 5-%d/%d" % (length-1, length))
 561             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 562         d.addCallback(_got)
 563         return d
 564
 565     def test_HEAD_FILEURL_range(self):
 566         headers = {"range": "bytes=1-10"}
 567         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 568                      return_response=True)
 569         def _got((res, status, headers)):
 570             self.failUnlessEqual(res, "")
 571             self.failUnlessEqual(int(status), 206)
 572             self.failUnless(headers.has_key("content-range"))
 573             self.failUnlessEqual(headers["content-range"][0],
 574                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 575         d.addCallback(_got)
 576         return d
 577
 578     def test_HEAD_FILEURL_partial_range(self):
 579         headers = {"range": "bytes=5-"}
 580         length  = len(self.BAR_CONTENTS)
 581         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 582                      return_response=True)
 583         def _got((res, status, headers)):
 584             self.failUnlessEqual(int(status), 206)
 585             self.failUnless(headers.has_key("content-range"))
 586             self.failUnlessEqual(headers["content-range"][0],
 587                                  "bytes 5-%d/%d" % (length-1, length))
 588         d.addCallback(_got)
 589         return d
 590
 591     def test_GET_FILEURL_range_bad(self):
 592         headers = {"range": "BOGUS=fizbop-quarnak"}
 593         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 594                              "400 Bad Request",
 595                              "Syntactically invalid http range header",
 596                              self.GET, self.public_url + "/foo/bar.txt",
 597                              headers=headers)
 598         return d
 599
 600     def test_HEAD_FILEURL(self):
 601         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 602         def _got((res, status, headers)):
 603             self.failUnlessEqual(res, "")
 604             self.failUnlessEqual(headers["content-length"][0],
 605                                  str(len(self.BAR_CONTENTS)))
 606             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 607         d.addCallback(_got)
 608         return d
 609
 610     def test_GET_FILEURL_named(self):
 611         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 612         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 613         d = self.GET(base + "/@@name=/blah.txt")
 614         d.addCallback(self.failUnlessIsBarDotTxt)
 615         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 616         d.addCallback(self.failUnlessIsBarDotTxt)
 617         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 618         d.addCallback(self.failUnlessIsBarDotTxt)
 619         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         save_url = base + "?save=true&filename=blah.txt"
 622         d.addCallback(lambda res: self.GET(save_url))
 623         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 624         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 625         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 626         u_url = base + "?save=true&filename=" + u_fn_e
 627         d.addCallback(lambda res: self.GET(u_url))
 628         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 629         return d
 630
 631     def test_PUT_FILEURL_named_bad(self):
 632         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 633         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 634                              "400 Bad Request",
 635                              "/file can only be used with GET or HEAD",
 636                              self.PUT, base + "/@@name=/blah.txt", "")
 637         return d
 638
 639     def test_GET_DIRURL_named_bad(self):
 640         base = "/file/%s" % urllib.quote(self._foo_uri)
 641         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 642                              "400 Bad Request",
 643                              "is not a file-cap",
 644                              self.GET, base + "/@@name=/blah.txt")
 645         return d
 646
 647     def test_GET_slash_file_bad(self):
 648         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 649                              "404 Not Found",
 650                              "/file must be followed by a file-cap and a name",
 651                              self.GET, "/file")
 652         return d
 653
 654     def test_GET_unhandled_URI_named(self):
 655         contents, n, newuri = self.makefile(12)
 656         verifier_cap = n.get_verify_cap().to_string()
 657         base = "/file/%s" % urllib.quote(verifier_cap)
 658         # client.create_node_from_uri() can't handle verify-caps
 659         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 660                              "400 Bad Request",
 661                              "is not a valid file- or directory- cap",
 662                              self.GET, base)
 663         return d
 664
 665     def test_GET_unhandled_URI(self):
 666         contents, n, newuri = self.makefile(12)
 667         verifier_cap = n.get_verify_cap().to_string()
 668         base = "/uri/%s" % urllib.quote(verifier_cap)
 669         # client.create_node_from_uri() can't handle verify-caps
 670         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 671                              "400 Bad Request",
 672                              "is not a valid file- or directory- cap",
 673                              self.GET, base)
 674         return d
 675
 676     def test_GET_FILE_URI(self):
 677         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 678         d = self.GET(base)
 679         d.addCallback(self.failUnlessIsBarDotTxt)
 680         return d
 681
 682     def test_GET_FILE_URI_badchild(self):
 683         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 684         errmsg = "Files have no children, certainly not named 'boguschild'"
 685         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 686                              "400 Bad Request", errmsg,
 687                              self.GET, base)
 688         return d
 689
 690     def test_PUT_FILE_URI_badchild(self):
 691         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 692         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 693         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 694                              "400 Bad Request", errmsg,
 695                              self.PUT, base, "")
 696         return d
 697
 698     def test_GET_FILEURL_save(self):
 699         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 700         # TODO: look at the headers, expect a Content-Disposition: attachment
 701         # header.
 702         d.addCallback(self.failUnlessIsBarDotTxt)
 703         return d
 704
 705     def test_GET_FILEURL_missing(self):
 706         d = self.GET(self.public_url + "/foo/missing")
 707         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 708         return d
 709
 710     def test_PUT_NEWFILEURL(self):
 711         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 712         # TODO: we lose the response code, so we can't check this
 713         #self.failUnlessEqual(responsecode, 201)
 714         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 715         d.addCallback(lambda res:
 716                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 717                                                       self.NEWFILE_CONTENTS))
 718         return d
 719
 720     def test_PUT_NEWFILEURL_range_bad(self):
 721         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 722         target = self.public_url + "/foo/new.txt"
 723         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 724                              "501 Not Implemented",
 725                              "Content-Range in PUT not yet supported",
 726                              # (and certainly not for immutable files)
 727                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 728                              headers=headers)
 729         d.addCallback(lambda res:
 730                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 731         return d
 732
 733     def test_PUT_NEWFILEURL_mutable(self):
 734         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 735                      self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 201)
 738         def _check_uri(res):
 739             u = uri.from_string_mutable_filenode(res)
 740             self.failUnless(u.is_mutable())
 741             self.failIf(u.is_readonly())
 742             return res
 743         d.addCallback(_check_uri)
 744         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 745         d.addCallback(lambda res:
 746                       self.failUnlessMutableChildContentsAre(self._foo_node,
 747                                                              u"new.txt",
 748                                                              self.NEWFILE_CONTENTS))
 749         return d
 750
 751     def test_PUT_NEWFILEURL_mutable_toobig(self):
 752         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 753                              "413 Request Entity Too Large",
 754                              "SDMF is limited to one segment, and 10001 > 10000",
 755                              self.PUT,
 756                              self.public_url + "/foo/new.txt?mutable=true",
 757                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 758         return d
 759
 760     def test_PUT_NEWFILEURL_replace(self):
 761         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 762         # TODO: we lose the response code, so we can't check this
 763         #self.failUnlessEqual(responsecode, 200)
 764         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 765         d.addCallback(lambda res:
 766                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 767                                                       self.NEWFILE_CONTENTS))
 768         return d
 769
 770     def test_PUT_NEWFILEURL_bad_t(self):
 771         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 772                              "PUT to a file: bad t=bogus",
 773                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 774                              "contents")
 775         return d
 776
 777     def test_PUT_NEWFILEURL_no_replace(self):
 778         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 779                      self.NEWFILE_CONTENTS)
 780         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 781                   "409 Conflict",
 782                   "There was already a child by that name, and you asked me "
 783                   "to not replace it")
 784         return d
 785
 786     def test_PUT_NEWFILEURL_mkdirs(self):
 787         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 788         fn = self._foo_node
 789         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 790         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 791         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 792         d.addCallback(lambda res:
 793                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 794                                                       self.NEWFILE_CONTENTS))
 795         return d
 796
 797     def test_PUT_NEWFILEURL_blocked(self):
 798         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 799                      self.NEWFILE_CONTENTS)
 800         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 801                   "409 Conflict",
 802                   "Unable to create directory 'blockingfile': a file was in the way")
 803         return d
 804
 805     def test_DELETE_FILEURL(self):
 806         d = self.DELETE(self.public_url + "/foo/bar.txt")
 807         d.addCallback(lambda res:
 808                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 809         return d
 810
 811     def test_DELETE_FILEURL_missing(self):
 812         d = self.DELETE(self.public_url + "/foo/missing")
 813         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 814         return d
 815
 816     def test_DELETE_FILEURL_missing2(self):
 817         d = self.DELETE(self.public_url + "/missing/missing")
 818         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 819         return d
 820
 821     def test_GET_FILEURL_json(self):
 822         # twisted.web.http.parse_qs ignores any query args without an '=', so
 823         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 824         # instead. This may make it tricky to emulate the S3 interface
 825         # completely.
 826         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 827         d.addCallback(self.failUnlessIsBarJSON)
 828         return d
 829
 830     def test_GET_FILEURL_json_missing(self):
 831         d = self.GET(self.public_url + "/foo/missing?json")
 832         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 833         return d
 834
 835     def test_GET_FILEURL_uri(self):
 836         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 837         def _check(res):
 838             self.failUnlessEqual(res, self._bar_txt_uri)
 839         d.addCallback(_check)
 840         d.addCallback(lambda res:
 841                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 842         def _check2(res):
 843             # for now, for files, uris and readonly-uris are the same
 844             self.failUnlessEqual(res, self._bar_txt_uri)
 845         d.addCallback(_check2)
 846         return d
 847
 848     def test_GET_FILEURL_badtype(self):
 849         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 850                                  "bad t=bogus",
 851                                  self.GET,
 852                                  self.public_url + "/foo/bar.txt?t=bogus")
 853         return d
 854
 855     def test_GET_FILEURL_uri_missing(self):
 856         d = self.GET(self.public_url + "/foo/missing?t=uri")
 857         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 858         return d
 859
 860     def test_GET_DIRURL(self):
 861         # the addSlash means we get a redirect here
 862         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 863         ROOT = "../../.."
 864         d = self.GET(self.public_url + "/foo", followRedirect=True)
 865         def _check(res):
 866             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 867                             in res, res)
 868             # the FILE reference points to a URI, but it should end in bar.txt
 869             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 870                        (ROOT, urllib.quote(self._bar_txt_uri)))
 871             get_bar = "".join([r'<td>',
 872                                r'<a href="%s">bar.txt</a>' % bar_url,
 873                                r'</td>',
 874                                r'\s+<td>FILE</td>',
 875                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 876                                ])
 877             self.failUnless(re.search(get_bar, res), res)
 878             for line in res.split("\n"):
 879                 # find the line that contains the delete button for bar.txt
 880                 if ("form action" in line and
 881                     'value="delete"' in line and
 882                     'value="bar.txt"' in line):
 883                     # the form target should use a relative URL
 884                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 885                     self.failUnless(('action="%s"' % foo_url) in line, line)
 886                     # and the when_done= should too
 887                     #done_url = urllib.quote(???)
 888                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 889                     #                in line, line)
 890                     break
 891             else:
 892                 self.fail("unable to find delete-bar.txt line", res)
 893
 894             # the DIR reference just points to a URI
 895             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 896             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 897                        + r'\s+<td>DIR</td>')
 898             self.failUnless(re.search(get_sub, res), res)
 899         d.addCallback(_check)
 900
 901         # look at a directory which is readonly
 902         d.addCallback(lambda res:
 903                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 904         def _check2(res):
 905             self.failUnless("(readonly)" in res, res)
 906             self.failIf("Upload a file" in res, res)
 907         d.addCallback(_check2)
 908
 909         # and at a directory that contains a readonly directory
 910         d.addCallback(lambda res:
 911                       self.GET(self.public_url, followRedirect=True))
 912         def _check3(res):
 913             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 914                                       '</td>\s+<td>DIR-RO</td>', res))
 915         d.addCallback(_check3)
 916
 917         # and an empty directory
 918         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 919         def _check4(res):
 920             self.failUnless("directory is empty!" in res, res)
 921         d.addCallback(_check4)
 922
 923         return d
 924
 925     def test_GET_DIRURL_badtype(self):
 926         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 927                                  400, "Bad Request",
 928                                  "bad t=bogus",
 929                                  self.GET,
 930                                  self.public_url + "/foo?t=bogus")
 931         return d
 932
 933     def test_GET_DIRURL_json(self):
 934         d = self.GET(self.public_url + "/foo?t=json")
 935         d.addCallback(self.failUnlessIsFooJSON)
 936         return d
 937
 938
 939     def test_POST_DIRURL_manifest_no_ophandle(self):
 940         d = self.shouldFail2(error.Error,
 941                              "test_POST_DIRURL_manifest_no_ophandle",
 942                              "400 Bad Request",
 943                              "slow operation requires ophandle=",
 944                              self.POST, self.public_url, t="start-manifest")
 945         return d
 946
 947     def test_POST_DIRURL_manifest(self):
 948         d = defer.succeed(None)
 949         def getman(ignored, output):
 950             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 951                           followRedirect=True)
 952             d.addCallback(self.wait_for_operation, "125")
 953             d.addCallback(self.get_operation_results, "125", output)
 954             return d
 955         d.addCallback(getman, None)
 956         def _got_html(manifest):
 957             self.failUnless("Manifest of SI=" in manifest)
 958             self.failUnless("<td>sub</td>" in manifest)
 959             self.failUnless(self._sub_uri in manifest)
 960             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 961         d.addCallback(_got_html)
 962
 963         # both t=status and unadorned GET should be identical
 964         d.addCallback(lambda res: self.GET("/operations/125"))
 965         d.addCallback(_got_html)
 966
 967         d.addCallback(getman, "html")
 968         d.addCallback(_got_html)
 969         d.addCallback(getman, "text")
 970         def _got_text(manifest):
 971             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 972             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 973         d.addCallback(_got_text)
 974         d.addCallback(getman, "JSON")
 975         def _got_json(res):
 976             data = res["manifest"]
 977             got = {}
 978             for (path_list, cap) in data:
 979                 got[tuple(path_list)] = cap
 980             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 981             self.failUnless((u"sub",u"baz.txt") in got)
 982             self.failUnless("finished" in res)
 983             self.failUnless("origin" in res)
 984             self.failUnless("storage-index" in res)
 985             self.failUnless("verifycaps" in res)
 986             self.failUnless("stats" in res)
 987         d.addCallback(_got_json)
 988         return d
 989
 990     def test_POST_DIRURL_deepsize_no_ophandle(self):
 991         d = self.shouldFail2(error.Error,
 992                              "test_POST_DIRURL_deepsize_no_ophandle",
 993                              "400 Bad Request",
 994                              "slow operation requires ophandle=",
 995                              self.POST, self.public_url, t="start-deep-size")
 996         return d
 997
 998     def test_POST_DIRURL_deepsize(self):
 999         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1000                       followRedirect=True)
1001         d.addCallback(self.wait_for_operation, "126")
1002         d.addCallback(self.get_operation_results, "126", "json")
1003         def _got_json(data):
1004             self.failUnlessEqual(data["finished"], True)
1005             size = data["size"]
1006             self.failUnless(size > 1000)
1007         d.addCallback(_got_json)
1008         d.addCallback(self.get_operation_results, "126", "text")
1009         def _got_text(res):
1010             mo = re.search(r'^size: (\d+)$', res, re.M)
1011             self.failUnless(mo, res)
1012             size = int(mo.group(1))
1013             # with directories, the size varies.
1014             self.failUnless(size > 1000)
1015         d.addCallback(_got_text)
1016         return d
1017
1018     def test_POST_DIRURL_deepstats_no_ophandle(self):
1019         d = self.shouldFail2(error.Error,
1020                              "test_POST_DIRURL_deepstats_no_ophandle",
1021                              "400 Bad Request",
1022                              "slow operation requires ophandle=",
1023                              self.POST, self.public_url, t="start-deep-stats")
1024         return d
1025
1026     def test_POST_DIRURL_deepstats(self):
1027         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1028                       followRedirect=True)
1029         d.addCallback(self.wait_for_operation, "127")
1030         d.addCallback(self.get_operation_results, "127", "json")
1031         def _got_json(stats):
1032             expected = {"count-immutable-files": 3,
1033                         "count-mutable-files": 0,
1034                         "count-literal-files": 0,
1035                         "count-files": 3,
1036                         "count-directories": 3,
1037                         "size-immutable-files": 57,
1038                         "size-literal-files": 0,
1039                         #"size-directories": 1912, # varies
1040                         #"largest-directory": 1590,
1041                         "largest-directory-children": 5,
1042                         "largest-immutable-file": 19,
1043                         }
1044             for k,v in expected.iteritems():
1045                 self.failUnlessEqual(stats[k], v,
1046                                      "stats[%s] was %s, not %s" %
1047                                      (k, stats[k], v))
1048             self.failUnlessEqual(stats["size-files-histogram"],
1049                                  [ [11, 31, 3] ])
1050         d.addCallback(_got_json)
1051         return d
1052
1053     def test_POST_DIRURL_stream_manifest(self):
1054         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1055         def _check(res):
1056             self.failUnless(res.endswith("\n"))
1057             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1058             self.failUnlessEqual(len(units), 7)
1059             self.failUnlessEqual(units[-1]["type"], "stats")
1060             first = units[0]
1061             self.failUnlessEqual(first["path"], [])
1062             self.failUnlessEqual(first["cap"], self._foo_uri)
1063             self.failUnlessEqual(first["type"], "directory")
1064             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1065             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1066             self.failIfEqual(baz["storage-index"], None)
1067             self.failIfEqual(baz["verifycap"], None)
1068             self.failIfEqual(baz["repaircap"], None)
1069             return
1070         d.addCallback(_check)
1071         return d
1072
1073     def test_GET_DIRURL_uri(self):
1074         d = self.GET(self.public_url + "/foo?t=uri")
1075         def _check(res):
1076             self.failUnlessEqual(res, self._foo_uri)
1077         d.addCallback(_check)
1078         return d
1079
1080     def test_GET_DIRURL_readonly_uri(self):
1081         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1082         def _check(res):
1083             self.failUnlessEqual(res, self._foo_readonly_uri)
1084         d.addCallback(_check)
1085         return d
1086
1087     def test_PUT_NEWDIRURL(self):
1088         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1089         d.addCallback(lambda res:
1090                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1091         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1092         d.addCallback(self.failUnlessNodeKeysAre, [])
1093         return d
1094
1095     def test_PUT_NEWDIRURL_exists(self):
1096         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1097         d.addCallback(lambda res:
1098                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1099         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1100         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1101         return d
1102
1103     def test_PUT_NEWDIRURL_blocked(self):
1104         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1105                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1106                              self.PUT,
1107                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1108         d.addCallback(lambda res:
1109                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1110         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1111         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1112         return d
1113
1114     def test_PUT_NEWDIRURL_mkdir_p(self):
1115         d = defer.succeed(None)
1116         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1117         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1118         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1119         def mkdir_p(mkpnode):
1120             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1121             d = self.POST(url)
1122             def made_subsub(ssuri):
1123                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1124                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1125                 d = self.POST(url)
1126                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1127                 return d
1128             d.addCallback(made_subsub)
1129             return d
1130         d.addCallback(mkdir_p)
1131         return d
1132
1133     def test_PUT_NEWDIRURL_mkdirs(self):
1134         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1135         d.addCallback(lambda res:
1136                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1137         d.addCallback(lambda res:
1138                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1139         d.addCallback(lambda res:
1140                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1141         d.addCallback(self.failUnlessNodeKeysAre, [])
1142         return d
1143
1144     def test_DELETE_DIRURL(self):
1145         d = self.DELETE(self.public_url + "/foo")
1146         d.addCallback(lambda res:
1147                       self.failIfNodeHasChild(self.public_root, u"foo"))
1148         return d
1149
1150     def test_DELETE_DIRURL_missing(self):
1151         d = self.DELETE(self.public_url + "/foo/missing")
1152         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1153         d.addCallback(lambda res:
1154                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1155         return d
1156
1157     def test_DELETE_DIRURL_missing2(self):
1158         d = self.DELETE(self.public_url + "/missing")
1159         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1160         return d
1161
1162     def dump_root(self):
1163         print "NODEWALK"
1164         w = webish.DirnodeWalkerMixin()
1165         def visitor(childpath, childnode, metadata):
1166             print childpath
1167         d = w.walk(self.public_root, visitor)
1168         return d
1169
1170     def failUnlessNodeKeysAre(self, node, expected_keys):
1171         for k in expected_keys:
1172             assert isinstance(k, unicode)
1173         d = node.list()
1174         def _check(children):
1175             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1176         d.addCallback(_check)
1177         return d
1178     def failUnlessNodeHasChild(self, node, name):
1179         assert isinstance(name, unicode)
1180         d = node.list()
1181         def _check(children):
1182             self.failUnless(name in children)
1183         d.addCallback(_check)
1184         return d
1185     def failIfNodeHasChild(self, node, name):
1186         assert isinstance(name, unicode)
1187         d = node.list()
1188         def _check(children):
1189             self.failIf(name in children)
1190         d.addCallback(_check)
1191         return d
1192
1193     def failUnlessChildContentsAre(self, node, name, expected_contents):
1194         assert isinstance(name, unicode)
1195         d = node.get_child_at_path(name)
1196         d.addCallback(lambda node: node.download_to_data())
1197         def _check(contents):
1198             self.failUnlessEqual(contents, expected_contents)
1199         d.addCallback(_check)
1200         return d
1201
1202     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1203         assert isinstance(name, unicode)
1204         d = node.get_child_at_path(name)
1205         d.addCallback(lambda node: node.download_best_version())
1206         def _check(contents):
1207             self.failUnlessEqual(contents, expected_contents)
1208         d.addCallback(_check)
1209         return d
1210
1211     def failUnlessChildURIIs(self, node, name, expected_uri):
1212         assert isinstance(name, unicode)
1213         d = node.get_child_at_path(name)
1214         def _check(child):
1215             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1216         d.addCallback(_check)
1217         return d
1218
1219     def failUnlessURIMatchesChild(self, got_uri, node, name):
1220         assert isinstance(name, unicode)
1221         d = node.get_child_at_path(name)
1222         def _check(child):
1223             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1224         d.addCallback(_check)
1225         return d
1226
1227     def failUnlessCHKURIHasContents(self, got_uri, contents):
1228         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1229
1230     def test_POST_upload(self):
1231         d = self.POST(self.public_url + "/foo", t="upload",
1232                       file=("new.txt", self.NEWFILE_CONTENTS))
1233         fn = self._foo_node
1234         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1235         d.addCallback(lambda res:
1236                       self.failUnlessChildContentsAre(fn, u"new.txt",
1237                                                       self.NEWFILE_CONTENTS))
1238         return d
1239
1240     def test_POST_upload_unicode(self):
1241         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1242         d = self.POST(self.public_url + "/foo", t="upload",
1243                       file=(filename, self.NEWFILE_CONTENTS))
1244         fn = self._foo_node
1245         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1246         d.addCallback(lambda res:
1247                       self.failUnlessChildContentsAre(fn, filename,
1248                                                       self.NEWFILE_CONTENTS))
1249         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1250         d.addCallback(lambda res: self.GET(target_url))
1251         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1252                                                             self.NEWFILE_CONTENTS,
1253                                                             contents))
1254         return d
1255
1256     def test_POST_upload_unicode_named(self):
1257         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1258         d = self.POST(self.public_url + "/foo", t="upload",
1259                       name=filename,
1260                       file=("overridden", self.NEWFILE_CONTENTS))
1261         fn = self._foo_node
1262         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1263         d.addCallback(lambda res:
1264                       self.failUnlessChildContentsAre(fn, filename,
1265                                                       self.NEWFILE_CONTENTS))
1266         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1267         d.addCallback(lambda res: self.GET(target_url))
1268         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1269                                                             self.NEWFILE_CONTENTS,
1270                                                             contents))
1271         return d
1272
1273     def test_POST_upload_no_link(self):
1274         d = self.POST("/uri", t="upload",
1275                       file=("new.txt", self.NEWFILE_CONTENTS))
1276         def _check_upload_results(page):
1277             # this should be a page which describes the results of the upload
1278             # that just finished.
1279             self.failUnless("Upload Results:" in page)
1280             self.failUnless("URI:" in page)
1281             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1282             mo = uri_re.search(page)
1283             self.failUnless(mo, page)
1284             new_uri = mo.group(1)
1285             return new_uri
1286         d.addCallback(_check_upload_results)
1287         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1288         return d
1289
1290     def test_POST_upload_no_link_whendone(self):
1291         d = self.POST("/uri", t="upload", when_done="/",
1292                       file=("new.txt", self.NEWFILE_CONTENTS))
1293         d.addBoth(self.shouldRedirect, "/")
1294         return d
1295
1296     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1297         d = defer.maybeDeferred(callable, *args, **kwargs)
1298         def done(res):
1299             if isinstance(res, failure.Failure):
1300                 res.trap(error.PageRedirect)
1301                 statuscode = res.value.status
1302                 target = res.value.location
1303                 return checker(statuscode, target)
1304             self.fail("%s: callable was supposed to redirect, not return '%s'"
1305                       % (which, res))
1306         d.addBoth(done)
1307         return d
1308
1309     def test_POST_upload_no_link_whendone_results(self):
1310         def check(statuscode, target):
1311             self.failUnlessEqual(statuscode, str(http.FOUND))
1312             self.failUnless(target.startswith(self.webish_url), target)
1313             return client.getPage(target, method="GET")
1314         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1315                                  check,
1316                                  self.POST, "/uri", t="upload",
1317                                  when_done="/uri/%(uri)s",
1318                                  file=("new.txt", self.NEWFILE_CONTENTS))
1319         d.addCallback(lambda res:
1320                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1321         return d
1322
1323     def test_POST_upload_no_link_mutable(self):
1324         d = self.POST("/uri", t="upload", mutable="true",
1325                       file=("new.txt", self.NEWFILE_CONTENTS))
1326         def _check(new_uri):
1327             new_uri = new_uri.strip()
1328             self.new_uri = new_uri
1329             u = IURI(new_uri)
1330             self.failUnless(IMutableFileURI.providedBy(u))
1331             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1332             n = self.s.create_node_from_uri(new_uri)
1333             return n.download_best_version()
1334         d.addCallback(_check)
1335         def _check2(data):
1336             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1337             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1338         d.addCallback(_check2)
1339         def _check3(data):
1340             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1341             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1342         d.addCallback(_check3)
1343         def _check4(data):
1344             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1345         d.addCallback(_check4)
1346         return d
1347
1348     def test_POST_upload_no_link_mutable_toobig(self):
1349         d = self.shouldFail2(error.Error,
1350                              "test_POST_upload_no_link_mutable_toobig",
1351                              "413 Request Entity Too Large",
1352                              "SDMF is limited to one segment, and 10001 > 10000",
1353                              self.POST,
1354                              "/uri", t="upload", mutable="true",
1355                              file=("new.txt",
1356                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1357         return d
1358
1359     def test_POST_upload_mutable(self):
1360         # this creates a mutable file
1361         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1362                       file=("new.txt", self.NEWFILE_CONTENTS))
1363         fn = self._foo_node
1364         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1365         d.addCallback(lambda res:
1366                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1367                                                              self.NEWFILE_CONTENTS))
1368         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1369         def _got(newnode):
1370             self.failUnless(IMutableFileNode.providedBy(newnode))
1371             self.failUnless(newnode.is_mutable())
1372             self.failIf(newnode.is_readonly())
1373             self._mutable_node = newnode
1374             self._mutable_uri = newnode.get_uri()
1375         d.addCallback(_got)
1376
1377         # now upload it again and make sure that the URI doesn't change
1378         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1379         d.addCallback(lambda res:
1380                       self.POST(self.public_url + "/foo", t="upload",
1381                                 mutable="true",
1382                                 file=("new.txt", NEWER_CONTENTS)))
1383         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1384         d.addCallback(lambda res:
1385                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1386                                                              NEWER_CONTENTS))
1387         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1388         def _got2(newnode):
1389             self.failUnless(IMutableFileNode.providedBy(newnode))
1390             self.failUnless(newnode.is_mutable())
1391             self.failIf(newnode.is_readonly())
1392             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1393         d.addCallback(_got2)
1394
1395         # upload a second time, using PUT instead of POST
1396         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1397         d.addCallback(lambda res:
1398                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1399         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1400         d.addCallback(lambda res:
1401                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1402                                                              NEW2_CONTENTS))
1403
1404         # finally list the directory, since mutable files are displayed
1405         # slightly differently
1406
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url + "/foo/",
1409                                followRedirect=True))
1410         def _check_page(res):
1411             # TODO: assert more about the contents
1412             self.failUnless("SSK" in res)
1413             return res
1414         d.addCallback(_check_page)
1415
1416         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1417         def _got3(newnode):
1418             self.failUnless(IMutableFileNode.providedBy(newnode))
1419             self.failUnless(newnode.is_mutable())
1420             self.failIf(newnode.is_readonly())
1421             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1422         d.addCallback(_got3)
1423
1424         # look at the JSON form of the enclosing directory
1425         d.addCallback(lambda res:
1426                       self.GET(self.public_url + "/foo/?t=json",
1427                                followRedirect=True))
1428         def _check_page_json(res):
1429             parsed = simplejson.loads(res)
1430             self.failUnlessEqual(parsed[0], "dirnode")
1431             children = dict( [(unicode(name),value)
1432                               for (name,value)
1433                               in parsed[1]["children"].iteritems()] )
1434             self.failUnless("new.txt" in children)
1435             new_json = children["new.txt"]
1436             self.failUnlessEqual(new_json[0], "filenode")
1437             self.failUnless(new_json[1]["mutable"])
1438             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1439             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1440             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1441         d.addCallback(_check_page_json)
1442
1443         # and the JSON form of the file
1444         d.addCallback(lambda res:
1445                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1446         def _check_file_json(res):
1447             parsed = simplejson.loads(res)
1448             self.failUnlessEqual(parsed[0], "filenode")
1449             self.failUnless(parsed[1]["mutable"])
1450             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1451             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1452             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1453         d.addCallback(_check_file_json)
1454
1455         # and look at t=uri and t=readonly-uri
1456         d.addCallback(lambda res:
1457                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1458         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1459         d.addCallback(lambda res:
1460                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1461         def _check_ro_uri(res):
1462             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1463             self.failUnlessEqual(res, ro_uri)
1464         d.addCallback(_check_ro_uri)
1465
1466         # make sure we can get to it from /uri/URI
1467         d.addCallback(lambda res:
1468                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1469         d.addCallback(lambda res:
1470                       self.failUnlessEqual(res, NEW2_CONTENTS))
1471
1472         # and that HEAD computes the size correctly
1473         d.addCallback(lambda res:
1474                       self.HEAD(self.public_url + "/foo/new.txt",
1475                                 return_response=True))
1476         def _got_headers((res, status, headers)):
1477             self.failUnlessEqual(res, "")
1478             self.failUnlessEqual(headers["content-length"][0],
1479                                  str(len(NEW2_CONTENTS)))
1480             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1481         d.addCallback(_got_headers)
1482
1483         # make sure that size errors are displayed correctly for overwrite
1484         d.addCallback(lambda res:
1485                       self.shouldFail2(error.Error,
1486                                        "test_POST_upload_mutable-toobig",
1487                                        "413 Request Entity Too Large",
1488                                        "SDMF is limited to one segment, and 10001 > 10000",
1489                                        self.POST,
1490                                        self.public_url + "/foo", t="upload",
1491                                        mutable="true",
1492                                        file=("new.txt",
1493                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1494                                        ))
1495
1496         d.addErrback(self.dump_error)
1497         return d
1498
1499     def test_POST_upload_mutable_toobig(self):
1500         d = self.shouldFail2(error.Error,
1501                              "test_POST_upload_no_link_mutable_toobig",
1502                              "413 Request Entity Too Large",
1503                              "SDMF is limited to one segment, and 10001 > 10000",
1504                              self.POST,
1505                              self.public_url + "/foo",
1506                              t="upload", mutable="true",
1507                              file=("new.txt",
1508                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1509         return d
1510
1511     def dump_error(self, f):
1512         # if the web server returns an error code (like 400 Bad Request),
1513         # web.client.getPage puts the HTTP response body into the .response
1514         # attribute of the exception object that it gives back. It does not
1515         # appear in the Failure's repr(), so the ERROR that trial displays
1516         # will be rather terse and unhelpful. addErrback this method to the
1517         # end of your chain to get more information out of these errors.
1518         if f.check(error.Error):
1519             print "web.error.Error:"
1520             print f
1521             print f.value.response
1522         return f
1523
1524     def test_POST_upload_replace(self):
1525         d = self.POST(self.public_url + "/foo", t="upload",
1526                       file=("bar.txt", self.NEWFILE_CONTENTS))
1527         fn = self._foo_node
1528         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1529         d.addCallback(lambda res:
1530                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1531                                                       self.NEWFILE_CONTENTS))
1532         return d
1533
1534     def test_POST_upload_no_replace_ok(self):
1535         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1536                       file=("new.txt", self.NEWFILE_CONTENTS))
1537         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1538         d.addCallback(lambda res: self.failUnlessEqual(res,
1539                                                        self.NEWFILE_CONTENTS))
1540         return d
1541
1542     def test_POST_upload_no_replace_queryarg(self):
1543         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1544                       file=("bar.txt", self.NEWFILE_CONTENTS))
1545         d.addBoth(self.shouldFail, error.Error,
1546                   "POST_upload_no_replace_queryarg",
1547                   "409 Conflict",
1548                   "There was already a child by that name, and you asked me "
1549                   "to not replace it")
1550         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1551         d.addCallback(self.failUnlessIsBarDotTxt)
1552         return d
1553
1554     def test_POST_upload_no_replace_field(self):
1555         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1556                       file=("bar.txt", self.NEWFILE_CONTENTS))
1557         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1558                   "409 Conflict",
1559                   "There was already a child by that name, and you asked me "
1560                   "to not replace it")
1561         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1562         d.addCallback(self.failUnlessIsBarDotTxt)
1563         return d
1564
1565     def test_POST_upload_whendone(self):
1566         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1567                       file=("new.txt", self.NEWFILE_CONTENTS))
1568         d.addBoth(self.shouldRedirect, "/THERE")
1569         fn = self._foo_node
1570         d.addCallback(lambda res:
1571                       self.failUnlessChildContentsAre(fn, u"new.txt",
1572                                                       self.NEWFILE_CONTENTS))
1573         return d
1574
1575     def test_POST_upload_named(self):
1576         fn = self._foo_node
1577         d = self.POST(self.public_url + "/foo", t="upload",
1578                       name="new.txt", file=self.NEWFILE_CONTENTS)
1579         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1580         d.addCallback(lambda res:
1581                       self.failUnlessChildContentsAre(fn, u"new.txt",
1582                                                       self.NEWFILE_CONTENTS))
1583         return d
1584
1585     def test_POST_upload_named_badfilename(self):
1586         d = self.POST(self.public_url + "/foo", t="upload",
1587                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1588         d.addBoth(self.shouldFail, error.Error,
1589                   "test_POST_upload_named_badfilename",
1590                   "400 Bad Request",
1591                   "name= may not contain a slash",
1592                   )
1593         # make sure that nothing was added
1594         d.addCallback(lambda res:
1595                       self.failUnlessNodeKeysAre(self._foo_node,
1596                                                  [u"bar.txt", u"blockingfile",
1597                                                   u"empty", u"n\u00fc.txt",
1598                                                   u"sub"]))
1599         return d
1600
1601     def test_POST_FILEURL_check(self):
1602         bar_url = self.public_url + "/foo/bar.txt"
1603         d = self.POST(bar_url, t="check")
1604         def _check(res):
1605             self.failUnless("Healthy :" in res)
1606         d.addCallback(_check)
1607         redir_url = "http://allmydata.org/TARGET"
1608         def _check2(statuscode, target):
1609             self.failUnlessEqual(statuscode, str(http.FOUND))
1610             self.failUnlessEqual(target, redir_url)
1611         d.addCallback(lambda res:
1612                       self.shouldRedirect2("test_POST_FILEURL_check",
1613                                            _check2,
1614                                            self.POST, bar_url,
1615                                            t="check",
1616                                            when_done=redir_url))
1617         d.addCallback(lambda res:
1618                       self.POST(bar_url, t="check", return_to=redir_url))
1619         def _check3(res):
1620             self.failUnless("Healthy :" in res)
1621             self.failUnless("Return to parent directory" in res)
1622             self.failUnless(redir_url in res)
1623         d.addCallback(_check3)
1624
1625         d.addCallback(lambda res:
1626                       self.POST(bar_url, t="check", output="JSON"))
1627         def _check_json(res):
1628             data = simplejson.loads(res)
1629             self.failUnless("storage-index" in data)
1630             self.failUnless(data["results"]["healthy"])
1631         d.addCallback(_check_json)
1632
1633         return d
1634
1635     def test_POST_FILEURL_check_and_repair(self):
1636         bar_url = self.public_url + "/foo/bar.txt"
1637         d = self.POST(bar_url, t="check", repair="true")
1638         def _check(res):
1639             self.failUnless("Healthy :" in res)
1640         d.addCallback(_check)
1641         redir_url = "http://allmydata.org/TARGET"
1642         def _check2(statuscode, target):
1643             self.failUnlessEqual(statuscode, str(http.FOUND))
1644             self.failUnlessEqual(target, redir_url)
1645         d.addCallback(lambda res:
1646                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1647                                            _check2,
1648                                            self.POST, bar_url,
1649                                            t="check", repair="true",
1650                                            when_done=redir_url))
1651         d.addCallback(lambda res:
1652                       self.POST(bar_url, t="check", return_to=redir_url))
1653         def _check3(res):
1654             self.failUnless("Healthy :" in res)
1655             self.failUnless("Return to parent directory" in res)
1656             self.failUnless(redir_url in res)
1657         d.addCallback(_check3)
1658         return d
1659
1660     def test_POST_DIRURL_check(self):
1661         foo_url = self.public_url + "/foo/"
1662         d = self.POST(foo_url, t="check")
1663         def _check(res):
1664             self.failUnless("Healthy :" in res, res)
1665         d.addCallback(_check)
1666         redir_url = "http://allmydata.org/TARGET"
1667         def _check2(statuscode, target):
1668             self.failUnlessEqual(statuscode, str(http.FOUND))
1669             self.failUnlessEqual(target, redir_url)
1670         d.addCallback(lambda res:
1671                       self.shouldRedirect2("test_POST_DIRURL_check",
1672                                            _check2,
1673                                            self.POST, foo_url,
1674                                            t="check",
1675                                            when_done=redir_url))
1676         d.addCallback(lambda res:
1677                       self.POST(foo_url, t="check", return_to=redir_url))
1678         def _check3(res):
1679             self.failUnless("Healthy :" in res, res)
1680             self.failUnless("Return to parent directory" in res)
1681             self.failUnless(redir_url in res)
1682         d.addCallback(_check3)
1683
1684         d.addCallback(lambda res:
1685                       self.POST(foo_url, t="check", output="JSON"))
1686         def _check_json(res):
1687             data = simplejson.loads(res)
1688             self.failUnless("storage-index" in data)
1689             self.failUnless(data["results"]["healthy"])
1690         d.addCallback(_check_json)
1691
1692         return d
1693
1694     def test_POST_DIRURL_check_and_repair(self):
1695         foo_url = self.public_url + "/foo/"
1696         d = self.POST(foo_url, t="check", repair="true")
1697         def _check(res):
1698             self.failUnless("Healthy :" in res, res)
1699         d.addCallback(_check)
1700         redir_url = "http://allmydata.org/TARGET"
1701         def _check2(statuscode, target):
1702             self.failUnlessEqual(statuscode, str(http.FOUND))
1703             self.failUnlessEqual(target, redir_url)
1704         d.addCallback(lambda res:
1705                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1706                                            _check2,
1707                                            self.POST, foo_url,
1708                                            t="check", repair="true",
1709                                            when_done=redir_url))
1710         d.addCallback(lambda res:
1711                       self.POST(foo_url, t="check", return_to=redir_url))
1712         def _check3(res):
1713             self.failUnless("Healthy :" in res)
1714             self.failUnless("Return to parent directory" in res)
1715             self.failUnless(redir_url in res)
1716         d.addCallback(_check3)
1717         return d
1718
1719     def wait_for_operation(self, ignored, ophandle):
1720         url = "/operations/" + ophandle
1721         url += "?t=status&output=JSON"
1722         d = self.GET(url)
1723         def _got(res):
1724             data = simplejson.loads(res)
1725             if not data["finished"]:
1726                 d = self.stall(delay=1.0)
1727                 d.addCallback(self.wait_for_operation, ophandle)
1728                 return d
1729             return data
1730         d.addCallback(_got)
1731         return d
1732
1733     def get_operation_results(self, ignored, ophandle, output=None):
1734         url = "/operations/" + ophandle
1735         url += "?t=status"
1736         if output:
1737             url += "&output=" + output
1738         d = self.GET(url)
1739         def _got(res):
1740             if output and output.lower() == "json":
1741                 return simplejson.loads(res)
1742             return res
1743         d.addCallback(_got)
1744         return d
1745
1746     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1747         d = self.shouldFail2(error.Error,
1748                              "test_POST_DIRURL_deepcheck_no_ophandle",
1749                              "400 Bad Request",
1750                              "slow operation requires ophandle=",
1751                              self.POST, self.public_url, t="start-deep-check")
1752         return d
1753
1754     def test_POST_DIRURL_deepcheck(self):
1755         def _check_redirect(statuscode, target):
1756             self.failUnlessEqual(statuscode, str(http.FOUND))
1757             self.failUnless(target.endswith("/operations/123"))
1758         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1759                                  self.POST, self.public_url,
1760                                  t="start-deep-check", ophandle="123")
1761         d.addCallback(self.wait_for_operation, "123")
1762         def _check_json(data):
1763             self.failUnlessEqual(data["finished"], True)
1764             self.failUnlessEqual(data["count-objects-checked"], 8)
1765             self.failUnlessEqual(data["count-objects-healthy"], 8)
1766         d.addCallback(_check_json)
1767         d.addCallback(self.get_operation_results, "123", "html")
1768         def _check_html(res):
1769             self.failUnless("Objects Checked: <span>8</span>" in res)
1770             self.failUnless("Objects Healthy: <span>8</span>" in res)
1771         d.addCallback(_check_html)
1772
1773         d.addCallback(lambda res:
1774                       self.GET("/operations/123/"))
1775         d.addCallback(_check_html) # should be the same as without the slash
1776
1777         d.addCallback(lambda res:
1778                       self.shouldFail2(error.Error, "one", "404 Not Found",
1779                                        "No detailed results for SI bogus",
1780                                        self.GET, "/operations/123/bogus"))
1781
1782         foo_si = self._foo_node.get_storage_index()
1783         foo_si_s = base32.b2a(foo_si)
1784         d.addCallback(lambda res:
1785                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1786         def _check_foo_json(res):
1787             data = simplejson.loads(res)
1788             self.failUnlessEqual(data["storage-index"], foo_si_s)
1789             self.failUnless(data["results"]["healthy"])
1790         d.addCallback(_check_foo_json)
1791         return d
1792
1793     def test_POST_DIRURL_deepcheck_and_repair(self):
1794         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1795                       ophandle="124", output="json", followRedirect=True)
1796         d.addCallback(self.wait_for_operation, "124")
1797         def _check_json(data):
1798             self.failUnlessEqual(data["finished"], True)
1799             self.failUnlessEqual(data["count-objects-checked"], 8)
1800             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1801             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1802             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1803             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1804             self.failUnlessEqual(data["count-repairs-successful"], 0)
1805             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1806             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1807             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1808             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1809         d.addCallback(_check_json)
1810         d.addCallback(self.get_operation_results, "124", "html")
1811         def _check_html(res):
1812             self.failUnless("Objects Checked: <span>8</span>" in res)
1813
1814             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1815             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1816             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1817
1818             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1819             self.failUnless("Repairs Successful: <span>0</span>" in res)
1820             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1821
1822             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1823             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1824             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1825         d.addCallback(_check_html)
1826         return d
1827
1828     def test_POST_FILEURL_bad_t(self):
1829         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1830                              "POST to file: bad t=bogus",
1831                              self.POST, self.public_url + "/foo/bar.txt",
1832                              t="bogus")
1833         return d
1834
1835     def test_POST_mkdir(self): # return value?
1836         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1837         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1838         d.addCallback(self.failUnlessNodeKeysAre, [])
1839         return d
1840
1841     def test_POST_mkdir_2(self):
1842         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1843         d.addCallback(lambda res:
1844                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1845         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1846         d.addCallback(self.failUnlessNodeKeysAre, [])
1847         return d
1848
1849     def test_POST_mkdirs_2(self):
1850         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1851         d.addCallback(lambda res:
1852                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1853         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1854         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1855         d.addCallback(self.failUnlessNodeKeysAre, [])
1856         return d
1857
1858     def test_POST_mkdir_no_parentdir_noredirect(self):
1859         d = self.POST("/uri?t=mkdir")
1860         def _after_mkdir(res):
1861             uri.NewDirectoryURI.init_from_string(res)
1862         d.addCallback(_after_mkdir)
1863         return d
1864
1865     def test_POST_mkdir_no_parentdir_redirect(self):
1866         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1867         d.addBoth(self.shouldRedirect, None, statuscode='303')
1868         def _check_target(target):
1869             target = urllib.unquote(target)
1870             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1871         d.addCallback(_check_target)
1872         return d
1873
1874     def test_POST_noparent_bad(self):
1875         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
1876                                  "/uri accepts only PUT, PUT?t=mkdir, "
1877                                  "POST?t=upload, and POST?t=mkdir",
1878                                  self.POST, "/uri?t=bogus")
1879         return d
1880
1881     def test_welcome_page_mkdir_button(self):
1882         # Fetch the welcome page.
1883         d = self.GET("/")
1884         def _after_get_welcome_page(res):
1885             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1886             mo = MKDIR_BUTTON_RE.search(res)
1887             formaction = mo.group(1)
1888             formt = mo.group(2)
1889             formaname = mo.group(3)
1890             formavalue = mo.group(4)
1891             return (formaction, formt, formaname, formavalue)
1892         d.addCallback(_after_get_welcome_page)
1893         def _after_parse_form(res):
1894             (formaction, formt, formaname, formavalue) = res
1895             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1896         d.addCallback(_after_parse_form)
1897         d.addBoth(self.shouldRedirect, None, statuscode='303')
1898         return d
1899
1900     def test_POST_mkdir_replace(self): # return value?
1901         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1902         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1903         d.addCallback(self.failUnlessNodeKeysAre, [])
1904         return d
1905
1906     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1907         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1908         d.addBoth(self.shouldFail, error.Error,
1909                   "POST_mkdir_no_replace_queryarg",
1910                   "409 Conflict",
1911                   "There was already a child by that name, and you asked me "
1912                   "to not replace it")
1913         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1914         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1915         return d
1916
1917     def test_POST_mkdir_no_replace_field(self): # return value?
1918         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1919                       replace="false")
1920         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1921                   "409 Conflict",
1922                   "There was already a child by that name, and you asked me "
1923                   "to not replace it")
1924         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1925         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1926         return d
1927
1928     def test_POST_mkdir_whendone_field(self):
1929         d = self.POST(self.public_url + "/foo",
1930                       t="mkdir", name="newdir", when_done="/THERE")
1931         d.addBoth(self.shouldRedirect, "/THERE")
1932         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1933         d.addCallback(self.failUnlessNodeKeysAre, [])
1934         return d
1935
1936     def test_POST_mkdir_whendone_queryarg(self):
1937         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1938                       t="mkdir", name="newdir")
1939         d.addBoth(self.shouldRedirect, "/THERE")
1940         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1941         d.addCallback(self.failUnlessNodeKeysAre, [])
1942         return d
1943
1944     def test_POST_bad_t(self):
1945         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1946                              "POST to a directory with bad t=BOGUS",
1947                              self.POST, self.public_url + "/foo", t="BOGUS")
1948         return d
1949
1950     def test_POST_set_children(self):
1951         contents9, n9, newuri9 = self.makefile(9)
1952         contents10, n10, newuri10 = self.makefile(10)
1953         contents11, n11, newuri11 = self.makefile(11)
1954
1955         reqbody = """{
1956                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1957                                                 "size": 0,
1958                                                 "metadata": {
1959                                                   "ctime": 1002777696.7564139,
1960                                                   "mtime": 1002777696.7564139
1961                                                  }
1962                                                } ],
1963                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1964                                                 "size": 1,
1965                                                 "metadata": {
1966                                                   "ctime": 1002777696.7564139,
1967                                                   "mtime": 1002777696.7564139
1968                                                  }
1969                                                } ],
1970                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1971                                                 "size": 2,
1972                                                 "metadata": {
1973                                                   "ctime": 1002777696.7564139,
1974                                                   "mtime": 1002777696.7564139
1975                                                  }
1976                                                } ]
1977                     }""" % (newuri9, newuri10, newuri11)
1978
1979         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1980
1981         d = client.getPage(url, method="POST", postdata=reqbody)
1982         def _then(res):
1983             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1984             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1985             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1986
1987         d.addCallback(_then)
1988         d.addErrback(self.dump_error)
1989         return d
1990
1991     def test_POST_put_uri(self):
1992         contents, n, newuri = self.makefile(8)
1993         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1994         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1995         d.addCallback(lambda res:
1996                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1997                                                       contents))
1998         return d
1999
2000     def test_POST_put_uri_replace(self):
2001         contents, n, newuri = self.makefile(8)
2002         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2003         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2004         d.addCallback(lambda res:
2005                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2006                                                       contents))
2007         return d
2008
2009     def test_POST_put_uri_no_replace_queryarg(self):
2010         contents, n, newuri = self.makefile(8)
2011         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2012                       name="bar.txt", uri=newuri)
2013         d.addBoth(self.shouldFail, error.Error,
2014                   "POST_put_uri_no_replace_queryarg",
2015                   "409 Conflict",
2016                   "There was already a child by that name, and you asked me "
2017                   "to not replace it")
2018         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2019         d.addCallback(self.failUnlessIsBarDotTxt)
2020         return d
2021
2022     def test_POST_put_uri_no_replace_field(self):
2023         contents, n, newuri = self.makefile(8)
2024         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2025                       name="bar.txt", uri=newuri)
2026         d.addBoth(self.shouldFail, error.Error,
2027                   "POST_put_uri_no_replace_field",
2028                   "409 Conflict",
2029                   "There was already a child by that name, and you asked me "
2030                   "to not replace it")
2031         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2032         d.addCallback(self.failUnlessIsBarDotTxt)
2033         return d
2034
2035     def test_POST_delete(self):
2036         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2037         d.addCallback(lambda res: self._foo_node.list())
2038         def _check(children):
2039             self.failIf(u"bar.txt" in children)
2040         d.addCallback(_check)
2041         return d
2042
2043     def test_POST_rename_file(self):
2044         d = self.POST(self.public_url + "/foo", t="rename",
2045                       from_name="bar.txt", to_name='wibble.txt')
2046         d.addCallback(lambda res:
2047                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2048         d.addCallback(lambda res:
2049                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2050         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2051         d.addCallback(self.failUnlessIsBarDotTxt)
2052         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2053         d.addCallback(self.failUnlessIsBarJSON)
2054         return d
2055
2056     def test_POST_rename_file_redundant(self):
2057         d = self.POST(self.public_url + "/foo", t="rename",
2058                       from_name="bar.txt", to_name='bar.txt')
2059         d.addCallback(lambda res:
2060                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2061         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2062         d.addCallback(self.failUnlessIsBarDotTxt)
2063         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2064         d.addCallback(self.failUnlessIsBarJSON)
2065         return d
2066
2067     def test_POST_rename_file_replace(self):
2068         # rename a file and replace a directory with it
2069         d = self.POST(self.public_url + "/foo", t="rename",
2070                       from_name="bar.txt", to_name='empty')
2071         d.addCallback(lambda res:
2072                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2073         d.addCallback(lambda res:
2074                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2075         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2076         d.addCallback(self.failUnlessIsBarDotTxt)
2077         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2078         d.addCallback(self.failUnlessIsBarJSON)
2079         return d
2080
2081     def test_POST_rename_file_no_replace_queryarg(self):
2082         # rename a file and replace a directory with it
2083         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2084                       from_name="bar.txt", to_name='empty')
2085         d.addBoth(self.shouldFail, error.Error,
2086                   "POST_rename_file_no_replace_queryarg",
2087                   "409 Conflict",
2088                   "There was already a child by that name, and you asked me "
2089                   "to not replace it")
2090         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2091         d.addCallback(self.failUnlessIsEmptyJSON)
2092         return d
2093
2094     def test_POST_rename_file_no_replace_field(self):
2095         # rename a file and replace a directory with it
2096         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2097                       from_name="bar.txt", to_name='empty')
2098         d.addBoth(self.shouldFail, error.Error,
2099                   "POST_rename_file_no_replace_field",
2100                   "409 Conflict",
2101                   "There was already a child by that name, and you asked me "
2102                   "to not replace it")
2103         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2104         d.addCallback(self.failUnlessIsEmptyJSON)
2105         return d
2106
2107     def failUnlessIsEmptyJSON(self, res):
2108         data = simplejson.loads(res)
2109         self.failUnlessEqual(data[0], "dirnode", data)
2110         self.failUnlessEqual(len(data[1]["children"]), 0)
2111
2112     def test_POST_rename_file_slash_fail(self):
2113         d = self.POST(self.public_url + "/foo", t="rename",
2114                       from_name="bar.txt", to_name='kirk/spock.txt')
2115         d.addBoth(self.shouldFail, error.Error,
2116                   "test_POST_rename_file_slash_fail",
2117                   "400 Bad Request",
2118                   "to_name= may not contain a slash",
2119                   )
2120         d.addCallback(lambda res:
2121                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2122         return d
2123
2124     def test_POST_rename_dir(self):
2125         d = self.POST(self.public_url, t="rename",
2126                       from_name="foo", to_name='plunk')
2127         d.addCallback(lambda res:
2128                       self.failIfNodeHasChild(self.public_root, u"foo"))
2129         d.addCallback(lambda res:
2130                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2131         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2132         d.addCallback(self.failUnlessIsFooJSON)
2133         return d
2134
2135     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2136         """ If target is not None then the redirection has to go to target.  If
2137         statuscode is not None then the redirection has to be accomplished with
2138         that HTTP status code."""
2139         if not isinstance(res, failure.Failure):
2140             to_where = (target is None) and "somewhere" or ("to " + target)
2141             self.fail("%s: we were expecting to get redirected %s, not get an"
2142                       " actual page: %s" % (which, to_where, res))
2143         res.trap(error.PageRedirect)
2144         if statuscode is not None:
2145             self.failUnlessEqual(res.value.status, statuscode,
2146                                  "%s: not a redirect" % which)
2147         if target is not None:
2148             # the PageRedirect does not seem to capture the uri= query arg
2149             # properly, so we can't check for it.
2150             realtarget = self.webish_url + target
2151             self.failUnlessEqual(res.value.location, realtarget,
2152                                  "%s: wrong target" % which)
2153         return res.value.location
2154
2155     def test_GET_URI_form(self):
2156         base = "/uri?uri=%s" % self._bar_txt_uri
2157         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2158         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2159         d = self.GET(base)
2160         d.addBoth(self.shouldRedirect, targetbase)
2161         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2162         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2163         d.addCallback(lambda res: self.GET(base+"&t=json"))
2164         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2165         d.addCallback(self.log, "about to get file by uri")
2166         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2167         d.addCallback(self.failUnlessIsBarDotTxt)
2168         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2169         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2170                                            followRedirect=True))
2171         d.addCallback(self.failUnlessIsFooJSON)
2172         d.addCallback(self.log, "got dir by uri")
2173
2174         return d
2175
2176     def test_GET_URI_form_bad(self):
2177         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2178                              "400 Bad Request", "GET /uri requires uri=",
2179                              self.GET, "/uri")
2180         return d
2181
2182     def test_GET_rename_form(self):
2183         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2184                      followRedirect=True)
2185         def _check(res):
2186             self.failUnless('name="when_done" value="."' in res, res)
2187             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2188         d.addCallback(_check)
2189         return d
2190
2191     def log(self, res, msg):
2192         #print "MSG: %s  RES: %s" % (msg, res)
2193         log.msg(msg)
2194         return res
2195
2196     def test_GET_URI_URL(self):
2197         base = "/uri/%s" % self._bar_txt_uri
2198         d = self.GET(base)
2199         d.addCallback(self.failUnlessIsBarDotTxt)
2200         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2201         d.addCallback(self.failUnlessIsBarDotTxt)
2202         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2203         d.addCallback(self.failUnlessIsBarDotTxt)
2204         return d
2205
2206     def test_GET_URI_URL_dir(self):
2207         base = "/uri/%s?t=json" % self._foo_uri
2208         d = self.GET(base)
2209         d.addCallback(self.failUnlessIsFooJSON)
2210         return d
2211
2212     def test_GET_URI_URL_missing(self):
2213         base = "/uri/%s" % self._bad_file_uri
2214         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2215                                  http.GONE, None, "NotEnoughSharesError",
2216                                  self.GET, base)
2217         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2218         # here? we must arrange for a download to fail after target.open()
2219         # has been called, and then inspect the response to see that it is
2220         # shorter than we expected.
2221         return d
2222
2223     def test_PUT_DIRURL_uri(self):
2224         d = self.s.create_empty_dirnode()
2225         def _made_dir(dn):
2226             new_uri = dn.get_uri()
2227             # replace /foo with a new (empty) directory
2228             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2229             d.addCallback(lambda res:
2230                           self.failUnlessEqual(res.strip(), new_uri))
2231             d.addCallback(lambda res:
2232                           self.failUnlessChildURIIs(self.public_root,
2233                                                     u"foo",
2234                                                     new_uri))
2235             return d
2236         d.addCallback(_made_dir)
2237         return d
2238
2239     def test_PUT_DIRURL_uri_noreplace(self):
2240         d = self.s.create_empty_dirnode()
2241         def _made_dir(dn):
2242             new_uri = dn.get_uri()
2243             # replace /foo with a new (empty) directory, but ask that
2244             # replace=false, so it should fail
2245             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2246                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2247                                  self.PUT,
2248                                  self.public_url + "/foo?t=uri&replace=false",
2249                                  new_uri)
2250             d.addCallback(lambda res:
2251                           self.failUnlessChildURIIs(self.public_root,
2252                                                     u"foo",
2253                                                     self._foo_uri))
2254             return d
2255         d.addCallback(_made_dir)
2256         return d
2257
2258     def test_PUT_DIRURL_bad_t(self):
2259         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2260                                  "400 Bad Request", "PUT to a directory",
2261                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2262         d.addCallback(lambda res:
2263                       self.failUnlessChildURIIs(self.public_root,
2264                                                 u"foo",
2265                                                 self._foo_uri))
2266         return d
2267
2268     def test_PUT_NEWFILEURL_uri(self):
2269         contents, n, new_uri = self.makefile(8)
2270         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2271         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2272         d.addCallback(lambda res:
2273                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2274                                                       contents))
2275         return d
2276
2277     def test_PUT_NEWFILEURL_uri_replace(self):
2278         contents, n, new_uri = self.makefile(8)
2279         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2280         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2281         d.addCallback(lambda res:
2282                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2283                                                       contents))
2284         return d
2285
2286     def test_PUT_NEWFILEURL_uri_no_replace(self):
2287         contents, n, new_uri = self.makefile(8)
2288         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2289         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2290                   "409 Conflict",
2291                   "There was already a child by that name, and you asked me "
2292                   "to not replace it")
2293         return d
2294
2295     def test_PUT_NEWFILE_URI(self):
2296         file_contents = "New file contents here\n"
2297         d = self.PUT("/uri", file_contents)
2298         def _check(uri):
2299             assert isinstance(uri, str), uri
2300             self.failUnless(uri in FakeCHKFileNode.all_contents)
2301             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2302                                  file_contents)
2303             return self.GET("/uri/%s" % uri)
2304         d.addCallback(_check)
2305         def _check2(res):
2306             self.failUnlessEqual(res, file_contents)
2307         d.addCallback(_check2)
2308         return d
2309
2310     def test_PUT_NEWFILE_URI_only_PUT(self):
2311         d = self.PUT("/uri?t=bogus", "")
2312         d.addBoth(self.shouldFail, error.Error,
2313                   "PUT_NEWFILE_URI_only_PUT",
2314                   "400 Bad Request",
2315                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2316         return d
2317
2318     def test_PUT_NEWFILE_URI_mutable(self):
2319         file_contents = "New file contents here\n"
2320         d = self.PUT("/uri?mutable=true", file_contents)
2321         def _check_mutable(uri):
2322             uri = uri.strip()
2323             u = IURI(uri)
2324             self.failUnless(IMutableFileURI.providedBy(u))
2325             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2326             n = self.s.create_node_from_uri(uri)
2327             return n.download_best_version()
2328         d.addCallback(_check_mutable)
2329         def _check2_mutable(data):
2330             self.failUnlessEqual(data, file_contents)
2331         d.addCallback(_check2_mutable)
2332         return d
2333
2334         def _check(uri):
2335             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2336             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2337                                  file_contents)
2338             return self.GET("/uri/%s" % uri)
2339         d.addCallback(_check)
2340         def _check2(res):
2341             self.failUnlessEqual(res, file_contents)
2342         d.addCallback(_check2)
2343         return d
2344
2345     def test_PUT_mkdir(self):
2346         d = self.PUT("/uri?t=mkdir", "")
2347         def _check(uri):
2348             n = self.s.create_node_from_uri(uri.strip())
2349             d2 = self.failUnlessNodeKeysAre(n, [])
2350             d2.addCallback(lambda res:
2351                            self.GET("/uri/%s?t=json" % uri))
2352             return d2
2353         d.addCallback(_check)
2354         d.addCallback(self.failUnlessIsEmptyJSON)
2355         return d
2356
2357     def test_POST_check(self):
2358         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2359         def _done(res):
2360             # this returns a string form of the results, which are probably
2361             # None since we're using fake filenodes.
2362             # TODO: verify that the check actually happened, by changing
2363             # FakeCHKFileNode to count how many times .check() has been
2364             # called.
2365             pass
2366         d.addCallback(_done)
2367         return d
2368
2369     def test_bad_method(self):
2370         url = self.webish_url + self.public_url + "/foo/bar.txt"
2371         d = self.shouldHTTPError("test_bad_method",
2372                                  501, "Not Implemented",
2373                                  "I don't know how to treat a BOGUS request.",
2374                                  client.getPage, url, method="BOGUS")
2375         return d
2376
2377     def test_short_url(self):
2378         url = self.webish_url + "/uri"
2379         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2380                                  "I don't know how to treat a DELETE request.",
2381                                  client.getPage, url, method="DELETE")
2382         return d
2383
2384     def test_ophandle_bad(self):
2385         url = self.webish_url + "/operations/bogus?t=status"
2386         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2387                                  "unknown/expired handle 'bogus'",
2388                                  client.getPage, url)
2389         return d
2390
2391     def test_ophandle_cancel(self):
2392         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2393                       followRedirect=True)
2394         d.addCallback(lambda ignored:
2395                       self.GET("/operations/128?t=status&output=JSON"))
2396         def _check1(res):
2397             data = simplejson.loads(res)
2398             self.failUnless("finished" in data, res)
2399             monitor = self.ws.root.child_operations.handles["128"][0]
2400             d = self.POST("/operations/128?t=cancel&output=JSON")
2401             def _check2(res):
2402                 data = simplejson.loads(res)
2403                 self.failUnless("finished" in data, res)
2404                 # t=cancel causes the handle to be forgotten
2405                 self.failUnless(monitor.is_cancelled())
2406             d.addCallback(_check2)
2407             return d
2408         d.addCallback(_check1)
2409         d.addCallback(lambda ignored:
2410                       self.shouldHTTPError("test_ophandle_cancel",
2411                                            404, "404 Not Found",
2412                                            "unknown/expired handle '128'",
2413                                            self.GET,
2414                                            "/operations/128?t=status&output=JSON"))
2415         return d
2416
2417     def test_ophandle_retainfor(self):
2418         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2419                       followRedirect=True)
2420         d.addCallback(lambda ignored:
2421                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2422         def _check1(res):
2423             data = simplejson.loads(res)
2424             self.failUnless("finished" in data, res)
2425         d.addCallback(_check1)
2426         # the retain-for=0 will cause the handle to be expired very soon
2427         d.addCallback(self.stall, 2.0)
2428         d.addCallback(lambda ignored:
2429                       self.shouldHTTPError("test_ophandle_retainfor",
2430                                            404, "404 Not Found",
2431                                            "unknown/expired handle '129'",
2432                                            self.GET,
2433                                            "/operations/129?t=status&output=JSON"))
2434         return d
2435
2436     def test_ophandle_release_after_complete(self):
2437         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2438                       followRedirect=True)
2439         d.addCallback(self.wait_for_operation, "130")
2440         d.addCallback(lambda ignored:
2441                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2442         # the release-after-complete=true will cause the handle to be expired
2443         d.addCallback(lambda ignored:
2444                       self.shouldHTTPError("test_ophandle_release_after_complete",
2445                                            404, "404 Not Found",
2446                                            "unknown/expired handle '130'",
2447                                            self.GET,
2448                                            "/operations/130?t=status&output=JSON"))
2449         return d
2450
2451     def test_incident(self):
2452         d = self.POST("/report_incident", details="eek")
2453         def _done(res):
2454             self.failUnless("Thank you for your report!" in res, res)
2455         d.addCallback(_done)
2456         return d
2457
2458     def test_static(self):
2459         webdir = os.path.join(self.staticdir, "subdir")
2460         fileutil.make_dirs(webdir)
2461         f = open(os.path.join(webdir, "hello.txt"), "wb")
2462         f.write("hello")
2463         f.close()
2464
2465         d = self.GET("/static/subdir/hello.txt")
2466         def _check(res):
2467             self.failUnlessEqual(res, "hello")
2468         d.addCallback(_check)
2469         return d
2470
2471
2472 class Util(unittest.TestCase):
2473     def test_abbreviate_time(self):
2474         self.failUnlessEqual(common.abbreviate_time(None), "")
2475         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2476         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2477         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2478         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2479
2480     def test_abbreviate_rate(self):
2481         self.failUnlessEqual(common.abbreviate_rate(None), "")
2482         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2483         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2484         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2485
2486     def test_abbreviate_size(self):
2487         self.failUnlessEqual(common.abbreviate_size(None), "")
2488         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2489         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2490         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2491         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2492
2493     def test_plural(self):
2494         def convert(s):
2495             return "%d second%s" % (s, status.plural(s))
2496         self.failUnlessEqual(convert(0), "0 seconds")
2497         self.failUnlessEqual(convert(1), "1 second")
2498         self.failUnlessEqual(convert(2), "2 seconds")
2499         def convert2(s):
2500             return "has share%s: %s" % (status.plural(s), ",".join(s))
2501         self.failUnlessEqual(convert2([]), "has shares: ")
2502         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2503         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2504
2505
2506 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2507
2508     def CHECK(self, ign, which, args, clientnum=0):
2509         fileurl = self.fileurls[which]
2510         url = fileurl + "?" + args
2511         return self.GET(url, method="POST", clientnum=clientnum)
2512
2513     def test_filecheck(self):
2514         self.basedir = "web/Grid/filecheck"
2515         self.set_up_grid()
2516         c0 = self.g.clients[0]
2517         self.uris = {}
2518         DATA = "data" * 100
2519         d = c0.upload(upload.Data(DATA, convergence=""))
2520         def _stash_uri(ur, which):
2521             self.uris[which] = ur.uri
2522         d.addCallback(_stash_uri, "good")
2523         d.addCallback(lambda ign:
2524                       c0.upload(upload.Data(DATA+"1", convergence="")))
2525         d.addCallback(_stash_uri, "sick")
2526         d.addCallback(lambda ign:
2527                       c0.upload(upload.Data(DATA+"2", convergence="")))
2528         d.addCallback(_stash_uri, "dead")
2529         def _stash_mutable_uri(n, which):
2530             self.uris[which] = n.get_uri()
2531             assert isinstance(self.uris[which], str)
2532         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2533         d.addCallback(_stash_mutable_uri, "corrupt")
2534         d.addCallback(lambda ign:
2535                       c0.upload(upload.Data("literal", convergence="")))
2536         d.addCallback(_stash_uri, "small")
2537
2538         def _compute_fileurls(ignored):
2539             self.fileurls = {}
2540             for which in self.uris:
2541                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2542         d.addCallback(_compute_fileurls)
2543
2544         def _clobber_shares(ignored):
2545             good_shares = self.find_shares(self.uris["good"])
2546             self.failUnlessEqual(len(good_shares), 10)
2547             sick_shares = self.find_shares(self.uris["sick"])
2548             os.unlink(sick_shares[0][2])
2549             dead_shares = self.find_shares(self.uris["dead"])
2550             for i in range(1, 10):
2551                 os.unlink(dead_shares[i][2])
2552             c_shares = self.find_shares(self.uris["corrupt"])
2553             cso = CorruptShareOptions()
2554             cso.stdout = StringIO()
2555             cso.parseOptions([c_shares[0][2]])
2556             corrupt_share(cso)
2557         d.addCallback(_clobber_shares)
2558
2559         d.addCallback(self.CHECK, "good", "t=check")
2560         def _got_html_good(res):
2561             self.failUnless("Healthy" in res, res)
2562             self.failIf("Not Healthy" in res, res)
2563         d.addCallback(_got_html_good)
2564         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2565         def _got_html_good_return_to(res):
2566             self.failUnless("Healthy" in res, res)
2567             self.failIf("Not Healthy" in res, res)
2568             self.failUnless('<a href="somewhere">Return to parent directory'
2569                             in res, res)
2570         d.addCallback(_got_html_good_return_to)
2571         d.addCallback(self.CHECK, "good", "t=check&output=json")
2572         def _got_json_good(res):
2573             r = simplejson.loads(res)
2574             self.failUnlessEqual(r["summary"], "Healthy")
2575             self.failUnless(r["results"]["healthy"])
2576             self.failIf(r["results"]["needs-rebalancing"])
2577             self.failUnless(r["results"]["recoverable"])
2578         d.addCallback(_got_json_good)
2579
2580         d.addCallback(self.CHECK, "small", "t=check")
2581         def _got_html_small(res):
2582             self.failUnless("Literal files are always healthy" in res, res)
2583             self.failIf("Not Healthy" in res, res)
2584         d.addCallback(_got_html_small)
2585         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2586         def _got_html_small_return_to(res):
2587             self.failUnless("Literal files are always healthy" in res, res)
2588             self.failIf("Not Healthy" in res, res)
2589             self.failUnless('<a href="somewhere">Return to parent directory'
2590                             in res, res)
2591         d.addCallback(_got_html_small_return_to)
2592         d.addCallback(self.CHECK, "small", "t=check&output=json")
2593         def _got_json_small(res):
2594             r = simplejson.loads(res)
2595             self.failUnlessEqual(r["storage-index"], "")
2596             self.failUnless(r["results"]["healthy"])
2597         d.addCallback(_got_json_small)
2598
2599         d.addCallback(self.CHECK, "sick", "t=check")
2600         def _got_html_sick(res):
2601             self.failUnless("Not Healthy" in res, res)
2602         d.addCallback(_got_html_sick)
2603         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2604         def _got_json_sick(res):
2605             r = simplejson.loads(res)
2606             self.failUnlessEqual(r["summary"],
2607                                  "Not Healthy: 9 shares (enc 3-of-10)")
2608             self.failIf(r["results"]["healthy"])
2609             self.failIf(r["results"]["needs-rebalancing"])
2610             self.failUnless(r["results"]["recoverable"])
2611         d.addCallback(_got_json_sick)
2612
2613         d.addCallback(self.CHECK, "dead", "t=check")
2614         def _got_html_dead(res):
2615             self.failUnless("Not Healthy" in res, res)
2616         d.addCallback(_got_html_dead)
2617         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2618         def _got_json_dead(res):
2619             r = simplejson.loads(res)
2620             self.failUnlessEqual(r["summary"],
2621                                  "Not Healthy: 1 shares (enc 3-of-10)")
2622             self.failIf(r["results"]["healthy"])
2623             self.failIf(r["results"]["needs-rebalancing"])
2624             self.failIf(r["results"]["recoverable"])
2625         d.addCallback(_got_json_dead)
2626
2627         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2628         def _got_html_corrupt(res):
2629             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2630         d.addCallback(_got_html_corrupt)
2631         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2632         def _got_json_corrupt(res):
2633             r = simplejson.loads(res)
2634             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2635                             r["summary"])
2636             self.failIf(r["results"]["healthy"])
2637             self.failUnless(r["results"]["recoverable"])
2638             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2639             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2640         d.addCallback(_got_json_corrupt)
2641
2642         d.addErrback(self.explain_web_error)
2643         return d
2644
2645     def test_repair_html(self):
2646         self.basedir = "web/Grid/repair_html"
2647         self.set_up_grid()
2648         c0 = self.g.clients[0]
2649         self.uris = {}
2650         DATA = "data" * 100
2651         d = c0.upload(upload.Data(DATA, convergence=""))
2652         def _stash_uri(ur, which):
2653             self.uris[which] = ur.uri
2654         d.addCallback(_stash_uri, "good")
2655         d.addCallback(lambda ign:
2656                       c0.upload(upload.Data(DATA+"1", convergence="")))
2657         d.addCallback(_stash_uri, "sick")
2658         d.addCallback(lambda ign:
2659                       c0.upload(upload.Data(DATA+"2", convergence="")))
2660         d.addCallback(_stash_uri, "dead")
2661         def _stash_mutable_uri(n, which):
2662             self.uris[which] = n.get_uri()
2663             assert isinstance(self.uris[which], str)
2664         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2665         d.addCallback(_stash_mutable_uri, "corrupt")
2666
2667         def _compute_fileurls(ignored):
2668             self.fileurls = {}
2669             for which in self.uris:
2670                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2671         d.addCallback(_compute_fileurls)
2672
2673         def _clobber_shares(ignored):
2674             good_shares = self.find_shares(self.uris["good"])
2675             self.failUnlessEqual(len(good_shares), 10)
2676             sick_shares = self.find_shares(self.uris["sick"])
2677             os.unlink(sick_shares[0][2])
2678             dead_shares = self.find_shares(self.uris["dead"])
2679             for i in range(1, 10):
2680                 os.unlink(dead_shares[i][2])
2681             c_shares = self.find_shares(self.uris["corrupt"])
2682             cso = CorruptShareOptions()
2683             cso.stdout = StringIO()
2684             cso.parseOptions([c_shares[0][2]])
2685             corrupt_share(cso)
2686         d.addCallback(_clobber_shares)
2687
2688         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2689         def _got_html_good(res):
2690             self.failUnless("Healthy" in res, res)
2691             self.failIf("Not Healthy" in res, res)
2692             self.failUnless("No repair necessary" in res, res)
2693         d.addCallback(_got_html_good)
2694
2695         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2696         def _got_html_sick(res):
2697             self.failUnless("Healthy : healthy" in res, res)
2698             self.failIf("Not Healthy" in res, res)
2699             self.failUnless("Repair successful" in res, res)
2700         d.addCallback(_got_html_sick)
2701
2702         # repair of a dead file will fail, of course, but it isn't yet
2703         # clear how this should be reported. Right now it shows up as
2704         # a "410 Gone".
2705         #
2706         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2707         #def _got_html_dead(res):
2708         #    print res
2709         #    self.failUnless("Healthy : healthy" in res, res)
2710         #    self.failIf("Not Healthy" in res, res)
2711         #    self.failUnless("No repair necessary" in res, res)
2712         #d.addCallback(_got_html_dead)
2713
2714         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2715         def _got_html_corrupt(res):
2716             self.failUnless("Healthy : Healthy" in res, res)
2717             self.failIf("Not Healthy" in res, res)
2718             self.failUnless("Repair successful" in res, res)
2719         d.addCallback(_got_html_corrupt)
2720
2721         d.addErrback(self.explain_web_error)
2722         return d
2723
2724     def test_repair_json(self):
2725         self.basedir = "web/Grid/repair_json"
2726         self.set_up_grid()
2727         c0 = self.g.clients[0]
2728         self.uris = {}
2729         DATA = "data" * 100
2730         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2731         def _stash_uri(ur, which):
2732             self.uris[which] = ur.uri
2733         d.addCallback(_stash_uri, "sick")
2734
2735         def _compute_fileurls(ignored):
2736             self.fileurls = {}
2737             for which in self.uris:
2738                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2739         d.addCallback(_compute_fileurls)
2740
2741         def _clobber_shares(ignored):
2742             sick_shares = self.find_shares(self.uris["sick"])
2743             os.unlink(sick_shares[0][2])
2744         d.addCallback(_clobber_shares)
2745
2746         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2747         def _got_json_sick(res):
2748             r = simplejson.loads(res)
2749             self.failUnlessEqual(r["repair-attempted"], True)
2750             self.failUnlessEqual(r["repair-successful"], True)
2751             self.failUnlessEqual(r["pre-repair-results"]["summary"],
2752                                  "Not Healthy: 9 shares (enc 3-of-10)")
2753             self.failIf(r["pre-repair-results"]["results"]["healthy"])
2754             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2755             self.failUnless(r["post-repair-results"]["results"]["healthy"])
2756         d.addCallback(_got_json_sick)
2757
2758         d.addErrback(self.explain_web_error)
2759         return d
2760
2761     def test_deep_check(self):
2762         self.basedir = "web/Grid/deep_check"
2763         self.set_up_grid()
2764         c0 = self.g.clients[0]
2765         self.uris = {}
2766         self.fileurls = {}
2767         DATA = "data" * 100
2768         d = c0.create_empty_dirnode()
2769         def _stash_root_and_create_file(n):
2770             self.rootnode = n
2771             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2772             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2773         d.addCallback(_stash_root_and_create_file)
2774         def _stash_uri(fn, which):
2775             self.uris[which] = fn.get_uri()
2776             return fn
2777         d.addCallback(_stash_uri, "good")
2778         d.addCallback(lambda ign:
2779                       self.rootnode.add_file(u"small",
2780                                              upload.Data("literal",
2781                                                         convergence="")))
2782         d.addCallback(_stash_uri, "small")
2783         d.addCallback(lambda ign:
2784                       self.rootnode.add_file(u"sick",
2785                                              upload.Data(DATA+"1",
2786                                                         convergence="")))
2787         d.addCallback(_stash_uri, "sick")
2788
2789         def _clobber_shares(ignored):
2790             self.delete_shares_numbered(self.uris["sick"], [0,1])
2791         d.addCallback(_clobber_shares)
2792
2793         # root
2794         # root/good
2795         # root/small
2796         # root/sick
2797
2798         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2799         def _done(res):
2800             units = [simplejson.loads(line)
2801                      for line in res.splitlines()
2802                      if line]
2803             self.failUnlessEqual(len(units), 4+1)
2804             # should be parent-first
2805             u0 = units[0]
2806             self.failUnlessEqual(u0["path"], [])
2807             self.failUnlessEqual(u0["type"], "directory")
2808             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2809             u0cr = u0["check-results"]
2810             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2811
2812             ugood = [u for u in units
2813                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2814             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2815             ugoodcr = ugood["check-results"]
2816             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2817
2818             stats = units[-1]
2819             self.failUnlessEqual(stats["type"], "stats")
2820             s = stats["stats"]
2821             self.failUnlessEqual(s["count-immutable-files"], 2)
2822             self.failUnlessEqual(s["count-literal-files"], 1)
2823             self.failUnlessEqual(s["count-directories"], 1)
2824         d.addCallback(_done)
2825
2826         # now add root/subdir and root/subdir/grandchild, then make subdir
2827         # unrecoverable, then see what happens
2828
2829         d.addCallback(lambda ign:
2830                       self.rootnode.create_empty_directory(u"subdir"))
2831         d.addCallback(_stash_uri, "subdir")
2832         d.addCallback(lambda subdir_node:
2833                       subdir_node.add_file(u"grandchild",
2834                                            upload.Data(DATA+"2",
2835                                                        convergence="")))
2836         d.addCallback(_stash_uri, "grandchild")
2837
2838         d.addCallback(lambda ign:
2839                       self.delete_shares_numbered(self.uris["subdir"],
2840                                                   range(1, 10)))
2841
2842         # root
2843         # root/good
2844         # root/small
2845         # root/sick
2846         # root/subdir [unrecoverable]
2847         # root/subdir/grandchild
2848
2849         # how should a streaming-JSON API indicate fatal error?
2850         # answer: emit ERROR: instead of a JSON string
2851
2852         d.addCallback(self.CHECK, "root", "t=stream-manifest")
2853         def _check_broken_manifest(res):
2854             lines = res.splitlines()
2855             error_lines = [i
2856                            for (i,line) in enumerate(lines)
2857                            if line.startswith("ERROR:")]
2858             if not error_lines:
2859                 self.fail("no ERROR: in output: %s" % (res,))
2860             first_error = error_lines[0]
2861             error_line = lines[first_error]
2862             error_msg = lines[first_error+1:]
2863             error_msg_s = "\n".join(error_msg) + "\n"
2864             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
2865                               error_line)
2866             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
2867             units = [simplejson.loads(line) for line in lines[:first_error]]
2868             self.failUnlessEqual(len(units), 5) # includes subdir
2869             last_unit = units[-1]
2870             self.failUnlessEqual(last_unit["path"], ["subdir"])
2871         d.addCallback(_check_broken_manifest)
2872
2873         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2874         def _check_broken_deepcheck(res):
2875             lines = res.splitlines()
2876             error_lines = [i
2877                            for (i,line) in enumerate(lines)
2878                            if line.startswith("ERROR:")]
2879             if not error_lines:
2880                 self.fail("no ERROR: in output: %s" % (res,))
2881             first_error = error_lines[0]
2882             error_line = lines[first_error]
2883             error_msg = lines[first_error+1:]
2884             error_msg_s = "\n".join(error_msg) + "\n"
2885             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
2886                               error_line)
2887             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
2888             units = [simplejson.loads(line) for line in lines[:first_error]]
2889             self.failUnlessEqual(len(units), 5) # includes subdir
2890             last_unit = units[-1]
2891             self.failUnlessEqual(last_unit["path"], ["subdir"])
2892             r = last_unit["check-results"]["results"]
2893             self.failUnlessEqual(r["count-recoverable-versions"], 0)
2894             self.failUnlessEqual(r["count-shares-good"], 1)
2895             self.failUnlessEqual(r["recoverable"], False)
2896         d.addCallback(_check_broken_deepcheck)
2897
2898         d.addErrback(self.explain_web_error)
2899         return d
2900
2901     def test_deep_check_and_repair(self):
2902         self.basedir = "web/Grid/deep_check_and_repair"
2903         self.set_up_grid()
2904         c0 = self.g.clients[0]
2905         self.uris = {}
2906         self.fileurls = {}
2907         DATA = "data" * 100
2908         d = c0.create_empty_dirnode()
2909         def _stash_root_and_create_file(n):
2910             self.rootnode = n
2911             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2912             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2913         d.addCallback(_stash_root_and_create_file)
2914         def _stash_uri(fn, which):
2915             self.uris[which] = fn.get_uri()
2916         d.addCallback(_stash_uri, "good")
2917         d.addCallback(lambda ign:
2918                       self.rootnode.add_file(u"small",
2919                                              upload.Data("literal",
2920                                                         convergence="")))
2921         d.addCallback(_stash_uri, "small")
2922         d.addCallback(lambda ign:
2923                       self.rootnode.add_file(u"sick",
2924                                              upload.Data(DATA+"1",
2925                                                         convergence="")))
2926         d.addCallback(_stash_uri, "sick")
2927         #d.addCallback(lambda ign:
2928         #              self.rootnode.add_file(u"dead",
2929         #                                     upload.Data(DATA+"2",
2930         #                                                convergence="")))
2931         #d.addCallback(_stash_uri, "dead")
2932
2933         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2934         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2935         #d.addCallback(_stash_uri, "corrupt")
2936
2937         def _clobber_shares(ignored):
2938             good_shares = self.find_shares(self.uris["good"])
2939             self.failUnlessEqual(len(good_shares), 10)
2940             sick_shares = self.find_shares(self.uris["sick"])
2941             os.unlink(sick_shares[0][2])
2942             #dead_shares = self.find_shares(self.uris["dead"])
2943             #for i in range(1, 10):
2944             #    os.unlink(dead_shares[i][2])
2945
2946             #c_shares = self.find_shares(self.uris["corrupt"])
2947             #cso = CorruptShareOptions()
2948             #cso.stdout = StringIO()
2949             #cso.parseOptions([c_shares[0][2]])
2950             #corrupt_share(cso)
2951         d.addCallback(_clobber_shares)
2952
2953         # root
2954         # root/good   CHK, 10 shares
2955         # root/small  LIT
2956         # root/sick   CHK, 9 shares
2957
2958         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2959         def _done(res):
2960             units = [simplejson.loads(line)
2961                      for line in res.splitlines()
2962                      if line]
2963             self.failUnlessEqual(len(units), 4+1)
2964             # should be parent-first
2965             u0 = units[0]
2966             self.failUnlessEqual(u0["path"], [])
2967             self.failUnlessEqual(u0["type"], "directory")
2968             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2969             u0crr = u0["check-and-repair-results"]
2970             self.failUnlessEqual(u0crr["repair-attempted"], False)
2971             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2972
2973             ugood = [u for u in units
2974                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2975             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2976             ugoodcrr = ugood["check-and-repair-results"]
2977             self.failUnlessEqual(u0crr["repair-attempted"], False)
2978             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2979
2980             usick = [u for u in units
2981                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
2982             self.failUnlessEqual(usick["cap"], self.uris["sick"])
2983             usickcrr = usick["check-and-repair-results"]
2984             self.failUnlessEqual(usickcrr["repair-attempted"], True)
2985             self.failUnlessEqual(usickcrr["repair-successful"], True)
2986             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2987             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2988
2989             stats = units[-1]
2990             self.failUnlessEqual(stats["type"], "stats")
2991             s = stats["stats"]
2992             self.failUnlessEqual(s["count-immutable-files"], 2)
2993             self.failUnlessEqual(s["count-literal-files"], 1)
2994             self.failUnlessEqual(s["count-directories"], 1)
2995         d.addCallback(_done)
2996
2997         d.addErrback(self.explain_web_error)
2998         return d
2999
3000     def _count_leases(self, ignored, which):
3001         u = self.uris[which]
3002         shares = self.find_shares(u)
3003         lease_counts = []
3004         for shnum, serverid, fn in shares:
3005             sf = get_share_file(fn)
3006             num_leases = len(list(sf.get_leases()))
3007         lease_counts.append( (fn, num_leases) )
3008         return lease_counts
3009
3010     def _assert_leasecount(self, lease_counts, expected):
3011         for (fn, num_leases) in lease_counts:
3012             if num_leases != expected:
3013                 self.fail("expected %d leases, have %d, on %s" %
3014                           (expected, num_leases, fn))
3015
3016     def test_add_lease(self):
3017         self.basedir = "web/Grid/add_lease"
3018         self.set_up_grid(num_clients=2)
3019         c0 = self.g.clients[0]
3020         self.uris = {}
3021         DATA = "data" * 100
3022         d = c0.upload(upload.Data(DATA, convergence=""))
3023         def _stash_uri(ur, which):
3024             self.uris[which] = ur.uri
3025         d.addCallback(_stash_uri, "one")
3026         d.addCallback(lambda ign:
3027                       c0.upload(upload.Data(DATA+"1", convergence="")))
3028         d.addCallback(_stash_uri, "two")
3029         def _stash_mutable_uri(n, which):
3030             self.uris[which] = n.get_uri()
3031             assert isinstance(self.uris[which], str)
3032         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3033         d.addCallback(_stash_mutable_uri, "mutable")
3034
3035         def _compute_fileurls(ignored):
3036             self.fileurls = {}
3037             for which in self.uris:
3038                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3039         d.addCallback(_compute_fileurls)
3040
3041         d.addCallback(self._count_leases, "one")
3042         d.addCallback(self._assert_leasecount, 1)
3043         d.addCallback(self._count_leases, "two")
3044         d.addCallback(self._assert_leasecount, 1)
3045         d.addCallback(self._count_leases, "mutable")
3046         d.addCallback(self._assert_leasecount, 1)
3047
3048         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3049         def _got_html_good(res):
3050             self.failUnless("Healthy" in res, res)
3051             self.failIf("Not Healthy" in res, res)
3052         d.addCallback(_got_html_good)
3053
3054         d.addCallback(self._count_leases, "one")
3055         d.addCallback(self._assert_leasecount, 1)
3056         d.addCallback(self._count_leases, "two")
3057         d.addCallback(self._assert_leasecount, 1)
3058         d.addCallback(self._count_leases, "mutable")
3059         d.addCallback(self._assert_leasecount, 1)
3060
3061         # this CHECK uses the original client, which uses the same
3062         # lease-secrets, so it will just renew the original lease
3063         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3064         d.addCallback(_got_html_good)
3065
3066         d.addCallback(self._count_leases, "one")
3067         d.addCallback(self._assert_leasecount, 1)
3068         d.addCallback(self._count_leases, "two")
3069         d.addCallback(self._assert_leasecount, 1)
3070         d.addCallback(self._count_leases, "mutable")
3071         d.addCallback(self._assert_leasecount, 1)
3072
3073         # this CHECK uses an alternate client, which adds a second lease
3074         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3075         d.addCallback(_got_html_good)
3076
3077         d.addCallback(self._count_leases, "one")
3078         d.addCallback(self._assert_leasecount, 2)
3079         d.addCallback(self._count_leases, "two")
3080         d.addCallback(self._assert_leasecount, 1)
3081         d.addCallback(self._count_leases, "mutable")
3082         d.addCallback(self._assert_leasecount, 1)
3083
3084         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3085         d.addCallback(_got_html_good)
3086
3087         d.addCallback(self._count_leases, "one")
3088         d.addCallback(self._assert_leasecount, 2)
3089         d.addCallback(self._count_leases, "two")
3090         d.addCallback(self._assert_leasecount, 1)
3091         d.addCallback(self._count_leases, "mutable")
3092         d.addCallback(self._assert_leasecount, 1)
3093
3094         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3095                       clientnum=1)
3096         d.addCallback(_got_html_good)
3097
3098         d.addCallback(self._count_leases, "one")
3099         d.addCallback(self._assert_leasecount, 2)
3100         d.addCallback(self._count_leases, "two")
3101         d.addCallback(self._assert_leasecount, 1)
3102         d.addCallback(self._count_leases, "mutable")
3103         d.addCallback(self._assert_leasecount, 2)
3104
3105         d.addErrback(self.explain_web_error)
3106         return d
3107
3108     def test_deep_add_lease(self):
3109         self.basedir = "web/Grid/deep_add_lease"
3110         self.set_up_grid(num_clients=2)
3111         c0 = self.g.clients[0]
3112         self.uris = {}
3113         self.fileurls = {}
3114         DATA = "data" * 100
3115         d = c0.create_empty_dirnode()
3116         def _stash_root_and_create_file(n):
3117             self.rootnode = n
3118             self.uris["root"] = n.get_uri()
3119             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3120             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3121         d.addCallback(_stash_root_and_create_file)
3122         def _stash_uri(fn, which):
3123             self.uris[which] = fn.get_uri()
3124         d.addCallback(_stash_uri, "one")
3125         d.addCallback(lambda ign:
3126                       self.rootnode.add_file(u"small",
3127                                              upload.Data("literal",
3128                                                         convergence="")))
3129         d.addCallback(_stash_uri, "small")
3130
3131         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3132         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3133         d.addCallback(_stash_uri, "mutable")
3134
3135         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3136         def _done(res):
3137             units = [simplejson.loads(line)
3138                      for line in res.splitlines()
3139                      if line]
3140             # root, one, small, mutable,   stats
3141             self.failUnlessEqual(len(units), 4+1)
3142         d.addCallback(_done)
3143
3144         d.addCallback(self._count_leases, "root")
3145         d.addCallback(self._assert_leasecount, 1)
3146         d.addCallback(self._count_leases, "one")
3147         d.addCallback(self._assert_leasecount, 1)
3148         d.addCallback(self._count_leases, "mutable")
3149         d.addCallback(self._assert_leasecount, 1)
3150
3151         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3152         d.addCallback(_done)
3153
3154         d.addCallback(self._count_leases, "root")
3155         d.addCallback(self._assert_leasecount, 1)
3156         d.addCallback(self._count_leases, "one")
3157         d.addCallback(self._assert_leasecount, 1)
3158         d.addCallback(self._count_leases, "mutable")
3159         d.addCallback(self._assert_leasecount, 1)
3160
3161         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3162                       clientnum=1)
3163         d.addCallback(_done)
3164
3165         d.addCallback(self._count_leases, "root")
3166         d.addCallback(self._assert_leasecount, 2)
3167         d.addCallback(self._count_leases, "one")
3168         d.addCallback(self._assert_leasecount, 2)
3169         d.addCallback(self._count_leases, "mutable")
3170         d.addCallback(self._assert_leasecount, 2)
3171
3172         d.addErrback(self.explain_web_error)
3173         return d
3174
3175
3176     def test_exceptions(self):
3177         self.basedir = "web/Grid/exceptions"
3178         self.set_up_grid(num_clients=1, num_servers=2)
3179         c0 = self.g.clients[0]
3180         self.fileurls = {}
3181         DATA = "data" * 100
3182         d = c0.create_empty_dirnode()
3183         def _stash_root(n):
3184             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3185             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3186             return n
3187         d.addCallback(_stash_root)
3188         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3189         def _stash_bad(ur):
3190             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3191             self.delete_shares_numbered(ur.uri, range(1,10))
3192
3193             u = uri.from_string(ur.uri)
3194             u.key = testutil.flip_bit(u.key, 0)
3195             baduri = u.to_string()
3196             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3197         d.addCallback(_stash_bad)
3198         d.addCallback(lambda ign: c0.create_empty_dirnode())
3199         def _mangle_dirnode_1share(n):
3200             u = n.get_uri()
3201             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3202             self.fileurls["dir-1share-json"] = url + "?t=json"
3203             self.delete_shares_numbered(u, range(1,10))
3204         d.addCallback(_mangle_dirnode_1share)
3205         d.addCallback(lambda ign: c0.create_empty_dirnode())
3206         def _mangle_dirnode_0share(n):
3207             u = n.get_uri()
3208             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3209             self.fileurls["dir-0share-json"] = url + "?t=json"
3210             self.delete_shares_numbered(u, range(0,10))
3211         d.addCallback(_mangle_dirnode_0share)
3212
3213         # NotEnoughSharesError should be reported sensibly, with a
3214         # text/plain explanation of the problem, and perhaps some
3215         # information on which shares *could* be found.
3216
3217         d.addCallback(lambda ignored:
3218                       self.shouldHTTPError("GET unrecoverable",
3219                                            410, "Gone", "NotEnoughSharesError",
3220                                            self.GET, self.fileurls["0shares"]))
3221         def _check_zero_shares(body):
3222             self.failIf("<html>" in body, body)
3223             body = " ".join(body.strip().split())
3224             exp = ("NotEnoughSharesError: no shares could be found. "
3225                    "Zero shares usually indicates a corrupt URI, or that "
3226                    "no servers were connected, but it might also indicate "
3227                    "severe corruption. You should perform a filecheck on "
3228                    "this object to learn more.")
3229             self.failUnlessEqual(exp, body)
3230         d.addCallback(_check_zero_shares)
3231
3232
3233         d.addCallback(lambda ignored:
3234                       self.shouldHTTPError("GET 1share",
3235                                            410, "Gone", "NotEnoughSharesError",
3236                                            self.GET, self.fileurls["1share"]))
3237         def _check_one_share(body):
3238             self.failIf("<html>" in body, body)
3239             body = " ".join(body.strip().split())
3240             exp = ("NotEnoughSharesError: 1 share found, but we need "
3241                    "3 to recover the file. This indicates that some "
3242                    "servers were unavailable, or that shares have been "
3243                    "lost to server departure, hard drive failure, or disk "
3244                    "corruption. You should perform a filecheck on "
3245                    "this object to learn more.")
3246             self.failUnlessEqual(exp, body)
3247         d.addCallback(_check_one_share)
3248
3249         d.addCallback(lambda ignored:
3250                       self.shouldHTTPError("GET imaginary",
3251                                            404, "Not Found", None,
3252                                            self.GET, self.fileurls["imaginary"]))
3253         def _missing_child(body):
3254             self.failUnless("No such child: imaginary" in body, body)
3255         d.addCallback(_missing_child)
3256
3257         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
3258         def _check_0shares_dir_html(body):
3259             self.failUnless("<html>" in body, body)
3260             # we should see the regular page, but without the child table or
3261             # the dirops forms
3262             body = " ".join(body.strip().split())
3263             self.failUnlessIn('href="?t=info">More info on this directory',
3264                               body)
3265             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3266                    "could not be retrieved, because there were insufficient "
3267                    "good shares. This might indicate that no servers were "
3268                    "connected, insufficient servers were connected, the URI "
3269                    "was corrupt, or that shares have been lost due to server "
3270                    "departure, hard drive failure, or disk corruption. You "
3271                    "should perform a filecheck on this object to learn more.")
3272             self.failUnlessIn(exp, body)
3273             self.failUnlessIn("No upload forms: directory is unreadable", body)
3274         d.addCallback(_check_0shares_dir_html)
3275
3276         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
3277         def _check_1shares_dir_html(body):
3278             # at some point, we'll split UnrecoverableFileError into 0-shares
3279             # and some-shares like we did for immutable files (since there
3280             # are different sorts of advice to offer in each case). For now,
3281             # they present the same way.
3282             self.failUnless("<html>" in body, body)
3283             body = " ".join(body.strip().split())
3284             self.failUnlessIn('href="?t=info">More info on this directory',
3285                               body)
3286             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3287                    "could not be retrieved, because there were insufficient "
3288                    "good shares. This might indicate that no servers were "
3289                    "connected, insufficient servers were connected, the URI "
3290                    "was corrupt, or that shares have been lost due to server "
3291                    "departure, hard drive failure, or disk corruption. You "
3292                    "should perform a filecheck on this object to learn more.")
3293             self.failUnlessIn(exp, body)
3294             self.failUnlessIn("No upload forms: directory is unreadable", body)
3295         d.addCallback(_check_1shares_dir_html)
3296
3297         d.addCallback(lambda ignored:
3298                       self.shouldHTTPError("GET dir-0share-json",
3299                                            410, "Gone", "UnrecoverableFileError",
3300                                            self.GET,
3301                                            self.fileurls["dir-0share-json"]))
3302         def _check_unrecoverable_file(body):
3303             self.failIf("<html>" in body, body)
3304             body = " ".join(body.strip().split())
3305             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3306                    "could not be retrieved, because there were insufficient "
3307                    "good shares. This might indicate that no servers were "
3308                    "connected, insufficient servers were connected, the URI "
3309                    "was corrupt, or that shares have been lost due to server "
3310                    "departure, hard drive failure, or disk corruption. You "
3311                    "should perform a filecheck on this object to learn more.")
3312             self.failUnlessEqual(exp, body)
3313         d.addCallback(_check_unrecoverable_file)
3314
3315         d.addCallback(lambda ignored:
3316                       self.shouldHTTPError("GET dir-1share-json",
3317                                            410, "Gone", "UnrecoverableFileError",
3318                                            self.GET,
3319                                            self.fileurls["dir-1share-json"]))
3320         d.addCallback(_check_unrecoverable_file)
3321
3322         d.addCallback(lambda ignored:
3323                       self.shouldHTTPError("GET imaginary",
3324                                            404, "Not Found", None,
3325                                            self.GET, self.fileurls["imaginary"]))
3326
3327         # attach a webapi child that throws a random error, to test how it
3328         # gets rendered.
3329         w = c0.getServiceNamed("webish")
3330         w.root.putChild("ERRORBOOM", ErrorBoom())
3331
3332         d.addCallback(lambda ignored:
3333                       self.shouldHTTPError("GET errorboom_html",
3334                                            500, "Internal Server Error", None,
3335                                            self.GET, "ERRORBOOM"))
3336         def _internal_error_html(body):
3337             # test that a weird exception during a webapi operation with
3338             # Accept:*/* results in a text/html stack trace, while one
3339             # without that Accept: line gets us a text/plain stack trace
3340             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3341         d.addCallback(_internal_error_html)
3342
3343         d.addCallback(lambda ignored:
3344                       self.shouldHTTPError("GET errorboom_text",
3345                                            500, "Internal Server Error", None,
3346                                            self.GET, "ERRORBOOM",
3347                                            headers={"accept": ["text/plain"]}))
3348         def _internal_error_text(body):
3349             # test that a weird exception during a webapi operation with
3350             # Accept:*/* results in a text/html stack trace, while one
3351             # without that Accept: line gets us a text/plain stack trace
3352             self.failIf("<html>" in body, body)
3353             self.failUnless(body.startswith("Traceback "), body)
3354         d.addCallback(_internal_error_text)
3355
3356         def _flush_errors(res):
3357             # Trial: please ignore the CompletelyUnhandledError in the logs
3358             self.flushLoggedErrors(CompletelyUnhandledError)
3359             return res
3360         d.addBoth(_flush_errors)
3361
3362         return d
3363
3364 class CompletelyUnhandledError(Exception):
3365     pass
3366 class ErrorBoom(rend.Page):
3367     def beforeRender(self, ctx):
3368         raise CompletelyUnhandledError("whoops")