1 import os.path, re, urllib, time, cgi
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow.util import escapeToXML
15 from nevow import rend
17 from allmydata import interfaces, uri, webish, dirnode
18 from allmydata.storage.shares import get_share_file
19 from allmydata.storage_client import StorageFarmBroker, StubServer
20 from allmydata.immutable import upload
21 from allmydata.immutable.downloader.status import DownloadStatus
22 from allmydata.dirnode import DirectoryNode
23 from allmydata.nodemaker import NodeMaker
24 from allmydata.unknown import UnknownNode
25 from allmydata.web import status, common
26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
27 from allmydata.util import fileutil, base32, hashutil
28 from allmydata.util.consumer import download_to_data
29 from allmydata.util.netstring import split_netstring
30 from allmydata.util.encodingutil import to_str
31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
32 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
33 make_mutable_file_uri, create_mutable_filenode
34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
35 from allmydata.mutable import servermap, publish, retrieve
36 import allmydata.test.common_util as testutil
37 from allmydata.test.no_network import GridTestMixin
38 from allmydata.test.common_web import HTTPClientGETFactory, \
40 from allmydata.client import Client, SecretHolder
41 from allmydata.introducer import IntroducerNode
43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
44 # create a webserver that works against them
46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
55 class FakeStatsProvider:
57 stats = {'stats': {}, 'counters': {}}
60 class FakeNodeMaker(NodeMaker):
65 'max_segment_size':128*1024 # 1024=KiB
67 def _create_lit(self, cap):
68 return FakeCHKFileNode(cap, self.all_contents)
69 def _create_immutable(self, cap):
70 return FakeCHKFileNode(cap, self.all_contents)
71 def _create_mutable(self, cap):
72 return FakeMutableFileNode(None, None,
73 self.encoding_params, None,
74 self.all_contents).init_from_cap(cap)
75 def create_mutable_file(self, contents="", keysize=None,
76 version=SDMF_VERSION):
77 n = FakeMutableFileNode(None, None, self.encoding_params, None,
79 return n.create(contents, version=version)
81 class FakeUploader(service.Service):
84 helper_connected = False
86 def upload(self, uploadable):
87 d = uploadable.get_size()
88 d.addCallback(lambda size: uploadable.read(size))
91 n = create_chk_filenode(data, self.all_contents)
92 ur = upload.UploadResults(file_size=len(data),
99 uri_extension_data={},
100 uri_extension_hash="fake",
101 verifycapstr="fakevcap")
102 ur.set_uri(n.get_uri())
104 d.addCallback(_got_data)
107 def get_helper_info(self):
108 return (self.helper_furl, self.helper_connected)
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeDisplayableServer(StubServer):
175 def __init__(self, serverid, nickname):
176 StubServer.__init__(self, serverid)
177 self.announcement = {"my-version": "allmydata-tahoe-fake",
178 "service-name": "storage",
179 "nickname": nickname}
180 def is_connected(self):
182 def get_permutation_seed(self):
184 def get_remote_host(self):
186 def get_last_loss_time(self):
188 def get_announcement_time(self):
190 def get_announcement(self):
191 return self.announcement
192 def get_nickname(self):
193 return self.announcement["nickname"]
195 class FakeBucketCounter(object):
197 return {"last-complete-bucket-count": 0}
198 def get_progress(self):
199 return {"estimated-time-per-cycle": 0,
200 "cycle-in-progress": False,
201 "remaining-wait-time": 0}
203 class FakeLeaseChecker(object):
205 self.expiration_enabled = False
207 self.override_lease_duration = None
208 self.sharetypes_to_expire = {}
210 return {"history": None}
211 def get_progress(self):
212 return {"estimated-time-per-cycle": 0,
213 "cycle-in-progress": False,
214 "remaining-wait-time": 0}
216 class FakeStorageServer(service.MultiService):
218 def __init__(self, nodeid, nickname):
219 service.MultiService.__init__(self)
220 self.my_nodeid = nodeid
221 self.nickname = nickname
222 self.bucket_counter = FakeBucketCounter()
223 self.lease_checker = FakeLeaseChecker()
225 return {"storage_server.accepting_immutable_shares": False}
227 class FakeClient(Client):
229 # don't upcall to Client.__init__, since we only want to initialize a
231 service.MultiService.__init__(self)
232 self.all_contents = {}
233 self.nodeid = "fake_nodeid"
234 self.nickname = u"fake_nickname \u263A"
235 self.introducer_furl = "None"
236 self.stats_provider = FakeStatsProvider()
237 self._secret_holder = SecretHolder("lease secret", "convergence secret")
239 self.convergence = "some random string"
240 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
241 # fake knowledge of another server
242 self.storage_broker.test_add_server("other_nodeid",
243 FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
244 self.introducer_client = None
245 self.history = FakeHistory()
246 self.uploader = FakeUploader()
247 self.uploader.all_contents = self.all_contents
248 self.uploader.setServiceParent(self)
249 self.blacklist = None
250 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
253 self.nodemaker.all_contents = self.all_contents
254 self.mutable_file_default = SDMF_VERSION
255 self.addService(FakeStorageServer(self.nodeid, self.nickname))
257 def startService(self):
258 return service.MultiService.startService(self)
259 def stopService(self):
260 return service.MultiService.stopService(self)
262 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
264 class WebMixin(object):
266 self.s = FakeClient()
267 self.s.startService()
268 self.staticdir = self.mktemp()
270 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
272 self.ws.setServiceParent(self.s)
273 self.webish_port = self.ws.getPortnum()
274 self.webish_url = self.ws.getURL()
275 assert self.webish_url.endswith("/")
276 self.webish_url = self.webish_url[:-1] # these tests add their own /
278 l = [ self.s.create_dirnode() for x in range(6) ]
279 d = defer.DeferredList(l)
281 self.public_root = res[0][1]
282 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
283 self.public_url = "/uri/" + self.public_root.get_uri()
284 self.private_root = res[1][1]
288 self._foo_uri = foo.get_uri()
289 self._foo_readonly_uri = foo.get_readonly_uri()
290 self._foo_verifycap = foo.get_verify_cap().to_string()
291 # NOTE: we ignore the deferred on all set_uri() calls, because we
292 # know the fake nodes do these synchronously
293 self.public_root.set_uri(u"foo", foo.get_uri(),
294 foo.get_readonly_uri())
296 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
297 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
298 self._bar_txt_verifycap = n.get_verify_cap().to_string()
301 # XXX: Do we ever use this?
302 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
304 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
307 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
308 assert self._quux_txt_uri.startswith("URI:MDMF")
309 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
311 foo.set_uri(u"empty", res[3][1].get_uri(),
312 res[3][1].get_readonly_uri())
313 sub_uri = res[4][1].get_uri()
314 self._sub_uri = sub_uri
315 foo.set_uri(u"sub", sub_uri, sub_uri)
316 sub = self.s.create_node_from_uri(sub_uri)
319 _ign, n, blocking_uri = self.makefile(1)
320 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
322 # filenode to test for html encoding issues
323 self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
324 self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
325 self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
326 self._htmlname_escaped = escapeToXML(self._htmlname_raw)
327 self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
328 self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
329 self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
330 foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
332 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
333 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
334 # still think of it as an umlaut
335 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
337 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
338 self._baz_file_uri = baz_file
339 sub.set_uri(u"baz.txt", baz_file, baz_file)
341 _ign, n, self._bad_file_uri = self.makefile(3)
342 # this uri should not be downloadable
343 del self.s.all_contents[self._bad_file_uri]
346 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
347 rodir.get_readonly_uri())
348 rodir.set_uri(u"nor", baz_file, baz_file)
354 # public/foo/quux.txt
355 # public/foo/blockingfile
356 # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
359 # public/foo/sub/baz.txt
361 # public/reedownlee/nor
362 self.NEWFILE_CONTENTS = "newfile contents\n"
364 return foo.get_metadata_for(u"bar.txt")
366 def _got_metadata(metadata):
367 self._bar_txt_metadata = metadata
368 d.addCallback(_got_metadata)
371 def get_all_contents(self):
372 return self.s.all_contents
374 def makefile(self, number):
375 contents = "contents of file %s\n" % number
376 n = create_chk_filenode(contents, self.get_all_contents())
377 return contents, n, n.get_uri()
379 def makefile_mutable(self, number, mdmf=False):
380 contents = "contents of mutable file %s\n" % number
381 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
382 return contents, n, n.get_uri(), n.get_readonly_uri()
385 return self.s.stopService()
387 def failUnlessIsBarDotTxt(self, res):
388 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
390 def failUnlessIsQuuxDotTxt(self, res):
391 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
393 def failUnlessIsBazDotTxt(self, res):
394 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
396 def failUnlessIsSubBazDotTxt(self, res):
397 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
399 def failUnlessIsBarJSON(self, res):
400 data = simplejson.loads(res)
401 self.failUnless(isinstance(data, list))
402 self.failUnlessEqual(data[0], "filenode")
403 self.failUnless(isinstance(data[1], dict))
404 self.failIf(data[1]["mutable"])
405 self.failIfIn("rw_uri", data[1]) # immutable
406 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
407 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
408 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
410 def failUnlessIsQuuxJSON(self, res, readonly=False):
411 data = simplejson.loads(res)
412 self.failUnless(isinstance(data, list))
413 self.failUnlessEqual(data[0], "filenode")
414 self.failUnless(isinstance(data[1], dict))
416 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
418 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
419 self.failUnless(metadata['mutable'])
421 self.failIfIn("rw_uri", metadata)
423 self.failUnlessIn("rw_uri", metadata)
424 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
425 self.failUnlessIn("ro_uri", metadata)
426 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
427 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
429 def failUnlessIsFooJSON(self, res):
430 data = simplejson.loads(res)
431 self.failUnless(isinstance(data, list))
432 self.failUnlessEqual(data[0], "dirnode", res)
433 self.failUnless(isinstance(data[1], dict))
434 self.failUnless(data[1]["mutable"])
435 self.failUnlessIn("rw_uri", data[1]) # mutable
436 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
437 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
438 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
440 kidnames = sorted([unicode(n) for n in data[1]["children"]])
441 self.failUnlessEqual(kidnames,
442 [self._htmlname_unicode, u"bar.txt", u"baz.txt",
443 u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
444 kids = dict( [(unicode(name),value)
446 in data[1]["children"].iteritems()] )
447 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
448 self.failUnlessIn("metadata", kids[u"sub"][1])
449 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
450 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
451 self.failUnlessIn("linkcrtime", tahoe_md)
452 self.failUnlessIn("linkmotime", tahoe_md)
453 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
454 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
455 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
456 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
457 self._bar_txt_verifycap)
458 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
459 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
460 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
461 self._bar_txt_metadata["tahoe"]["linkcrtime"])
462 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
464 self.failUnlessIn("quux.txt", kids)
465 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
467 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
468 self._quux_txt_readonly_uri)
470 def GET(self, urlpath, followRedirect=False, return_response=False,
472 # if return_response=True, this fires with (data, statuscode,
473 # respheaders) instead of just data.
474 assert not isinstance(urlpath, unicode)
475 url = self.webish_url + urlpath
476 factory = HTTPClientGETFactory(url, method="GET",
477 followRedirect=followRedirect, **kwargs)
478 reactor.connectTCP("localhost", self.webish_port, factory)
481 return (data, factory.status, factory.response_headers)
483 d.addCallback(_got_data)
484 return factory.deferred
486 def HEAD(self, urlpath, return_response=False, **kwargs):
487 # this requires some surgery, because twisted.web.client doesn't want
488 # to give us back the response headers.
489 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
490 reactor.connectTCP("localhost", self.webish_port, factory)
493 return (data, factory.status, factory.response_headers)
495 d.addCallback(_got_data)
496 return factory.deferred
498 def PUT(self, urlpath, data, **kwargs):
499 url = self.webish_url + urlpath
500 return client.getPage(url, method="PUT", postdata=data, **kwargs)
502 def DELETE(self, urlpath):
503 url = self.webish_url + urlpath
504 return client.getPage(url, method="DELETE")
506 def POST(self, urlpath, followRedirect=False, **fields):
507 sepbase = "boogabooga"
511 form.append('Content-Disposition: form-data; name="_charset"')
515 for name, value in fields.iteritems():
516 if isinstance(value, tuple):
517 filename, value = value
518 form.append('Content-Disposition: form-data; name="%s"; '
519 'filename="%s"' % (name, filename.encode("utf-8")))
521 form.append('Content-Disposition: form-data; name="%s"' % name)
523 if isinstance(value, unicode):
524 value = value.encode("utf-8")
527 assert isinstance(value, str)
534 body = "\r\n".join(form) + "\r\n"
535 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
536 return self.POST2(urlpath, body, headers, followRedirect)
538 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
539 url = self.webish_url + urlpath
540 return client.getPage(url, method="POST", postdata=body,
541 headers=headers, followRedirect=followRedirect)
543 def shouldFail(self, res, expected_failure, which,
544 substring=None, response_substring=None):
545 if isinstance(res, failure.Failure):
546 res.trap(expected_failure)
548 self.failUnlessIn(substring, str(res), which)
549 if response_substring:
550 self.failUnlessIn(response_substring, res.value.response, which)
552 self.fail("%s was supposed to raise %s, not get '%s'" %
553 (which, expected_failure, res))
555 def shouldFail2(self, expected_failure, which, substring,
557 callable, *args, **kwargs):
558 assert substring is None or isinstance(substring, str)
559 assert response_substring is None or isinstance(response_substring, str)
560 d = defer.maybeDeferred(callable, *args, **kwargs)
562 if isinstance(res, failure.Failure):
563 res.trap(expected_failure)
565 self.failUnlessIn(substring, str(res),
566 "'%s' not in '%s' for test '%s'" % \
567 (substring, str(res), which))
568 if response_substring:
569 self.failUnlessIn(response_substring, res.value.response,
570 "'%s' not in '%s' for test '%s'" % \
571 (response_substring, res.value.response,
574 self.fail("%s was supposed to raise %s, not get '%s'" %
575 (which, expected_failure, res))
579 def should404(self, res, which):
580 if isinstance(res, failure.Failure):
581 res.trap(error.Error)
582 self.failUnlessReallyEqual(res.value.status, "404")
584 self.fail("%s was supposed to Error(404), not get '%s'" %
587 def should302(self, res, which):
588 if isinstance(res, failure.Failure):
589 res.trap(error.Error)
590 self.failUnlessReallyEqual(res.value.status, "302")
592 self.fail("%s was supposed to Error(302), not get '%s'" %
596 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
597 def test_create(self):
600 def test_welcome(self):
603 self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
604 self.failUnlessIn(FAVICON_MARKUP, res)
605 self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
606 self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
607 self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
608 res_u = res.decode('utf-8')
609 self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
610 self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
611 self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
613 self.s.basedir = 'web/test_welcome'
614 fileutil.make_dirs("web/test_welcome")
615 fileutil.make_dirs("web/test_welcome/private")
617 d.addCallback(_check)
620 def test_introducer_status(self):
621 class MockIntroducerClient(object):
622 def __init__(self, connected):
623 self.connected = connected
624 def connected_to_introducer(self):
625 return self.connected
627 d = defer.succeed(None)
629 # introducer not connected, unguessable furl
630 def _set_introducer_not_connected_unguessable(ign):
631 self.s.introducer_furl = "pb://someIntroducer/secret"
632 self.s.introducer_client = MockIntroducerClient(False)
634 d.addCallback(_set_introducer_not_connected_unguessable)
635 def _check_introducer_not_connected_unguessable(res):
636 html = res.replace('\n', ' ')
637 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
638 self.failIfIn('pb://someIntroducer/secret', html)
639 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
640 d.addCallback(_check_introducer_not_connected_unguessable)
642 # introducer connected, unguessable furl
643 def _set_introducer_connected_unguessable(ign):
644 self.s.introducer_furl = "pb://someIntroducer/secret"
645 self.s.introducer_client = MockIntroducerClient(True)
647 d.addCallback(_set_introducer_connected_unguessable)
648 def _check_introducer_connected_unguessable(res):
649 html = res.replace('\n', ' ')
650 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
651 self.failIfIn('pb://someIntroducer/secret', html)
652 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
653 d.addCallback(_check_introducer_connected_unguessable)
655 # introducer connected, guessable furl
656 def _set_introducer_connected_guessable(ign):
657 self.s.introducer_furl = "pb://someIntroducer/introducer"
658 self.s.introducer_client = MockIntroducerClient(True)
660 d.addCallback(_set_introducer_connected_guessable)
661 def _check_introducer_connected_guessable(res):
662 html = res.replace('\n', ' ')
663 self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
664 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
665 d.addCallback(_check_introducer_connected_guessable)
668 def test_helper_status(self):
669 d = defer.succeed(None)
671 # set helper furl to None
672 def _set_no_helper(ign):
673 self.s.uploader.helper_furl = None
675 d.addCallback(_set_no_helper)
676 def _check_no_helper(res):
677 html = res.replace('\n', ' ')
678 self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
679 d.addCallback(_check_no_helper)
681 # enable helper, not connected
682 def _set_helper_not_connected(ign):
683 self.s.uploader.helper_furl = "pb://someHelper/secret"
684 self.s.uploader.helper_connected = False
686 d.addCallback(_set_helper_not_connected)
687 def _check_helper_not_connected(res):
688 html = res.replace('\n', ' ')
689 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
690 self.failIfIn('pb://someHelper/secret', html)
691 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
692 d.addCallback(_check_helper_not_connected)
694 # enable helper, connected
695 def _set_helper_connected(ign):
696 self.s.uploader.helper_furl = "pb://someHelper/secret"
697 self.s.uploader.helper_connected = True
699 d.addCallback(_set_helper_connected)
700 def _check_helper_connected(res):
701 html = res.replace('\n', ' ')
702 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
703 self.failIfIn('pb://someHelper/secret', html)
704 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
705 d.addCallback(_check_helper_connected)
708 def test_storage(self):
709 d = self.GET("/storage")
711 self.failUnlessIn('Storage Server Status', res)
712 self.failUnlessIn(FAVICON_MARKUP, res)
713 res_u = res.decode('utf-8')
714 self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
715 d.addCallback(_check)
718 def test_status(self):
719 h = self.s.get_history()
720 dl_num = h.list_all_download_statuses()[0].get_counter()
721 ul_num = h.list_all_upload_statuses()[0].get_counter()
722 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
723 pub_num = h.list_all_publish_statuses()[0].get_counter()
724 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
725 d = self.GET("/status", followRedirect=True)
727 self.failUnlessIn('Recent and Active Operations', res)
728 self.failUnlessIn('"down-%d"' % dl_num, res)
729 self.failUnlessIn('"up-%d"' % ul_num, res)
730 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
731 self.failUnlessIn('"publish-%d"' % pub_num, res)
732 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
733 d.addCallback(_check)
734 d.addCallback(lambda res: self.GET("/status/?t=json"))
735 def _check_json(res):
736 data = simplejson.loads(res)
737 self.failUnless(isinstance(data, dict))
738 #active = data["active"]
739 # TODO: test more. We need a way to fake an active operation
741 d.addCallback(_check_json)
743 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
745 self.failUnlessIn("File Download Status", res)
746 d.addCallback(_check_dl)
747 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
748 def _check_dl_json(res):
749 data = simplejson.loads(res)
750 self.failUnless(isinstance(data, dict))
751 self.failUnlessIn("read", data)
752 self.failUnlessEqual(data["read"][0]["length"], 120)
753 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
754 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
755 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
756 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
757 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
758 # serverids[] keys are strings, since that's what JSON does, but
759 # we'd really like them to be ints
760 self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
761 self.failUnless(data["serverids"].has_key("1"),
762 str(data["serverids"]))
763 self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
764 str(data["serverids"]))
765 self.failUnlessEqual(data["server_info"][phwr_id]["short"],
767 self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
769 self.failUnlessIn("dyhb", data)
770 self.failUnlessIn("misc", data)
771 d.addCallback(_check_dl_json)
772 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
774 self.failUnlessIn("File Upload Status", res)
775 d.addCallback(_check_ul)
776 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
777 def _check_mapupdate(res):
778 self.failUnlessIn("Mutable File Servermap Update Status", res)
779 d.addCallback(_check_mapupdate)
780 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
781 def _check_publish(res):
782 self.failUnlessIn("Mutable File Publish Status", res)
783 d.addCallback(_check_publish)
784 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
785 def _check_retrieve(res):
786 self.failUnlessIn("Mutable File Retrieve Status", res)
787 d.addCallback(_check_retrieve)
791 def test_status_numbers(self):
792 drrm = status.DownloadResultsRendererMixin()
793 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
794 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
795 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
796 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
797 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
798 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
799 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
800 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
801 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
803 urrm = status.UploadResultsRendererMixin()
804 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
805 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
806 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
807 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
808 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
809 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
810 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
811 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
812 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
814 def test_GET_FILEURL(self):
815 d = self.GET(self.public_url + "/foo/bar.txt")
816 d.addCallback(self.failUnlessIsBarDotTxt)
819 def test_GET_FILEURL_range(self):
820 headers = {"range": "bytes=1-10"}
821 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
822 return_response=True)
823 def _got((res, status, headers)):
824 self.failUnlessReallyEqual(int(status), 206)
825 self.failUnless(headers.has_key("content-range"))
826 self.failUnlessReallyEqual(headers["content-range"][0],
827 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
828 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
832 def test_GET_FILEURL_partial_range(self):
833 headers = {"range": "bytes=5-"}
834 length = len(self.BAR_CONTENTS)
835 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
836 return_response=True)
837 def _got((res, status, headers)):
838 self.failUnlessReallyEqual(int(status), 206)
839 self.failUnless(headers.has_key("content-range"))
840 self.failUnlessReallyEqual(headers["content-range"][0],
841 "bytes 5-%d/%d" % (length-1, length))
842 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
846 def test_GET_FILEURL_partial_end_range(self):
847 headers = {"range": "bytes=-5"}
848 length = len(self.BAR_CONTENTS)
849 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
850 return_response=True)
851 def _got((res, status, headers)):
852 self.failUnlessReallyEqual(int(status), 206)
853 self.failUnless(headers.has_key("content-range"))
854 self.failUnlessReallyEqual(headers["content-range"][0],
855 "bytes %d-%d/%d" % (length-5, length-1, length))
856 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
860 def test_GET_FILEURL_partial_range_overrun(self):
861 headers = {"range": "bytes=100-200"}
862 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
863 "416 Requested Range not satisfiable",
864 "First beyond end of file",
865 self.GET, self.public_url + "/foo/bar.txt",
869 def test_HEAD_FILEURL_range(self):
870 headers = {"range": "bytes=1-10"}
871 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
872 return_response=True)
873 def _got((res, status, headers)):
874 self.failUnlessReallyEqual(res, "")
875 self.failUnlessReallyEqual(int(status), 206)
876 self.failUnless(headers.has_key("content-range"))
877 self.failUnlessReallyEqual(headers["content-range"][0],
878 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
882 def test_HEAD_FILEURL_partial_range(self):
883 headers = {"range": "bytes=5-"}
884 length = len(self.BAR_CONTENTS)
885 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
886 return_response=True)
887 def _got((res, status, headers)):
888 self.failUnlessReallyEqual(int(status), 206)
889 self.failUnless(headers.has_key("content-range"))
890 self.failUnlessReallyEqual(headers["content-range"][0],
891 "bytes 5-%d/%d" % (length-1, length))
895 def test_HEAD_FILEURL_partial_end_range(self):
896 headers = {"range": "bytes=-5"}
897 length = len(self.BAR_CONTENTS)
898 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
899 return_response=True)
900 def _got((res, status, headers)):
901 self.failUnlessReallyEqual(int(status), 206)
902 self.failUnless(headers.has_key("content-range"))
903 self.failUnlessReallyEqual(headers["content-range"][0],
904 "bytes %d-%d/%d" % (length-5, length-1, length))
908 def test_HEAD_FILEURL_partial_range_overrun(self):
909 headers = {"range": "bytes=100-200"}
910 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
911 "416 Requested Range not satisfiable",
913 self.HEAD, self.public_url + "/foo/bar.txt",
917 def test_GET_FILEURL_range_bad(self):
918 headers = {"range": "BOGUS=fizbop-quarnak"}
919 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
920 return_response=True)
921 def _got((res, status, headers)):
922 self.failUnlessReallyEqual(int(status), 200)
923 self.failUnless(not headers.has_key("content-range"))
924 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
928 def test_HEAD_FILEURL(self):
929 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
930 def _got((res, status, headers)):
931 self.failUnlessReallyEqual(res, "")
932 self.failUnlessReallyEqual(headers["content-length"][0],
933 str(len(self.BAR_CONTENTS)))
934 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
938 def test_GET_FILEURL_named(self):
939 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
940 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
941 d = self.GET(base + "/@@name=/blah.txt")
942 d.addCallback(self.failUnlessIsBarDotTxt)
943 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
944 d.addCallback(self.failUnlessIsBarDotTxt)
945 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
946 d.addCallback(self.failUnlessIsBarDotTxt)
947 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
948 d.addCallback(self.failUnlessIsBarDotTxt)
949 save_url = base + "?save=true&filename=blah.txt"
950 d.addCallback(lambda res: self.GET(save_url))
951 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
952 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
953 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
954 u_url = base + "?save=true&filename=" + u_fn_e
955 d.addCallback(lambda res: self.GET(u_url))
956 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
959 def test_PUT_FILEURL_named_bad(self):
960 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
961 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
963 "/file can only be used with GET or HEAD",
964 self.PUT, base + "/@@name=/blah.txt", "")
968 def test_GET_DIRURL_named_bad(self):
969 base = "/file/%s" % urllib.quote(self._foo_uri)
970 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
973 self.GET, base + "/@@name=/blah.txt")
976 def test_GET_slash_file_bad(self):
977 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
979 "/file must be followed by a file-cap and a name",
983 def test_GET_unhandled_URI_named(self):
984 contents, n, newuri = self.makefile(12)
985 verifier_cap = n.get_verify_cap().to_string()
986 base = "/file/%s" % urllib.quote(verifier_cap)
987 # client.create_node_from_uri() can't handle verify-caps
988 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
989 "400 Bad Request", "is not a file-cap",
993 def test_GET_unhandled_URI(self):
994 contents, n, newuri = self.makefile(12)
995 verifier_cap = n.get_verify_cap().to_string()
996 base = "/uri/%s" % urllib.quote(verifier_cap)
997 # client.create_node_from_uri() can't handle verify-caps
998 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1000 "GET unknown URI type: can only do t=info",
1004 def test_GET_FILE_URI(self):
1005 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1007 d.addCallback(self.failUnlessIsBarDotTxt)
1010 def test_GET_FILE_URI_mdmf(self):
1011 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1013 d.addCallback(self.failUnlessIsQuuxDotTxt)
1016 def test_GET_FILE_URI_mdmf_extensions(self):
1017 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1019 d.addCallback(self.failUnlessIsQuuxDotTxt)
1022 def test_GET_FILE_URI_mdmf_readonly(self):
1023 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1025 d.addCallback(self.failUnlessIsQuuxDotTxt)
1028 def test_GET_FILE_URI_badchild(self):
1029 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1030 errmsg = "Files have no children, certainly not named 'boguschild'"
1031 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1032 "400 Bad Request", errmsg,
1036 def test_PUT_FILE_URI_badchild(self):
1037 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1038 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1039 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1040 "400 Bad Request", errmsg,
1044 def test_PUT_FILE_URI_mdmf(self):
1045 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1046 self._quux_new_contents = "new_contents"
1048 d.addCallback(lambda res:
1049 self.failUnlessIsQuuxDotTxt(res))
1050 d.addCallback(lambda ignored:
1051 self.PUT(base, self._quux_new_contents))
1052 d.addCallback(lambda ignored:
1054 d.addCallback(lambda res:
1055 self.failUnlessReallyEqual(res, self._quux_new_contents))
1058 def test_PUT_FILE_URI_mdmf_extensions(self):
1059 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1060 self._quux_new_contents = "new_contents"
1062 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1063 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1064 d.addCallback(lambda ignored: self.GET(base))
1065 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1069 def test_PUT_FILE_URI_mdmf_readonly(self):
1070 # We're not allowed to PUT things to a readonly cap.
1071 base = "/uri/%s" % self._quux_txt_readonly_uri
1073 d.addCallback(lambda res:
1074 self.failUnlessIsQuuxDotTxt(res))
1075 # What should we get here? We get a 500 error now; that's not right.
1076 d.addCallback(lambda ignored:
1077 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1078 "400 Bad Request", "read-only cap",
1079 self.PUT, base, "new data"))
1082 def test_PUT_FILE_URI_sdmf_readonly(self):
1083 # We're not allowed to put things to a readonly cap.
1084 base = "/uri/%s" % self._baz_txt_readonly_uri
1086 d.addCallback(lambda res:
1087 self.failUnlessIsBazDotTxt(res))
1088 d.addCallback(lambda ignored:
1089 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1090 "400 Bad Request", "read-only cap",
1091 self.PUT, base, "new_data"))
1094 def test_GET_etags(self):
1096 def _check_etags(uri):
1098 d2 = _get_etag(uri, 'json')
1099 d = defer.DeferredList([d1, d2], consumeErrors=True)
1100 def _check(results):
1101 # All deferred must succeed
1102 self.failUnless(all([r[0] for r in results]))
1103 # the etag for the t=json form should be just like the etag
1104 # fo the default t='' form, but with a 'json' suffix
1105 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1106 d.addCallback(_check)
1109 def _get_etag(uri, t=''):
1110 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1111 d = self.GET(targetbase, return_response=True, followRedirect=True)
1112 def _just_the_etag(result):
1113 data, response, headers = result
1114 etag = headers['etag'][0]
1115 if uri.startswith('URI:DIR'):
1116 self.failUnless(etag.startswith('DIR:'), etag)
1118 return d.addCallback(_just_the_etag)
1120 # Check that etags work with immutable directories
1121 (newkids, caps) = self._create_immutable_children()
1122 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1123 simplejson.dumps(newkids))
1124 def _stash_immdir_uri(uri):
1125 self._immdir_uri = uri
1127 d.addCallback(_stash_immdir_uri)
1128 d.addCallback(_check_etags)
1130 # Check that etags work with immutable files
1131 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1133 # use the ETag on GET
1134 def _check_match(ign):
1135 uri = "/uri/%s" % self._bar_txt_uri
1136 d = self.GET(uri, return_response=True)
1138 d.addCallback(lambda (data, code, headers):
1140 # do a GET that's supposed to match the ETag
1141 d.addCallback(lambda etag:
1142 self.GET(uri, return_response=True,
1143 headers={"If-None-Match": etag}))
1144 # make sure it short-circuited (304 instead of 200)
1145 d.addCallback(lambda (data, code, headers):
1146 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1148 d.addCallback(_check_match)
1150 def _no_etag(uri, t):
1151 target = "/uri/%s?t=%s" % (uri, t)
1152 d = self.GET(target, return_response=True, followRedirect=True)
1153 d.addCallback(lambda (data, code, headers):
1154 self.failIf("etag" in headers, target))
1156 def _yes_etag(uri, t):
1157 target = "/uri/%s?t=%s" % (uri, t)
1158 d = self.GET(target, return_response=True, followRedirect=True)
1159 d.addCallback(lambda (data, code, headers):
1160 self.failUnless("etag" in headers, target))
1163 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1164 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1165 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1166 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1167 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1169 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1170 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1171 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1172 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1173 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1174 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1178 # TODO: version of this with a Unicode filename
1179 def test_GET_FILEURL_save(self):
1180 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1181 return_response=True)
1182 def _got((res, statuscode, headers)):
1183 content_disposition = headers["content-disposition"][0]
1184 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1185 self.failUnlessIsBarDotTxt(res)
1189 def test_GET_FILEURL_missing(self):
1190 d = self.GET(self.public_url + "/foo/missing")
1191 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1194 def test_GET_FILEURL_info_mdmf(self):
1195 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1197 self.failUnlessIn("mutable file (mdmf)", res)
1198 self.failUnlessIn(self._quux_txt_uri, res)
1199 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1203 def test_GET_FILEURL_info_mdmf_readonly(self):
1204 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1206 self.failUnlessIn("mutable file (mdmf)", res)
1207 self.failIfIn(self._quux_txt_uri, res)
1208 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1212 def test_GET_FILEURL_info_sdmf(self):
1213 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1215 self.failUnlessIn("mutable file (sdmf)", res)
1216 self.failUnlessIn(self._baz_txt_uri, res)
1220 def test_GET_FILEURL_info_mdmf_extensions(self):
1221 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1223 self.failUnlessIn("mutable file (mdmf)", res)
1224 self.failUnlessIn(self._quux_txt_uri, res)
1225 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1229 def test_PUT_overwrite_only_files(self):
1230 # create a directory, put a file in that directory.
1231 contents, n, filecap = self.makefile(8)
1232 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1233 d.addCallback(lambda res:
1234 self.PUT(self.public_url + "/foo/dir/file1.txt",
1235 self.NEWFILE_CONTENTS))
1236 # try to overwrite the file with replace=only-files
1237 # (this should work)
1238 d.addCallback(lambda res:
1239 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1241 d.addCallback(lambda res:
1242 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1243 "There was already a child by that name, and you asked me "
1244 "to not replace it",
1245 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1249 def test_PUT_NEWFILEURL(self):
1250 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1251 # TODO: we lose the response code, so we can't check this
1252 #self.failUnlessReallyEqual(responsecode, 201)
1253 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1254 d.addCallback(lambda res:
1255 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1256 self.NEWFILE_CONTENTS))
1259 def test_PUT_NEWFILEURL_not_mutable(self):
1260 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1261 self.NEWFILE_CONTENTS)
1262 # TODO: we lose the response code, so we can't check this
1263 #self.failUnlessReallyEqual(responsecode, 201)
1264 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1265 d.addCallback(lambda res:
1266 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1267 self.NEWFILE_CONTENTS))
1270 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1271 # this should get us a few segments of an MDMF mutable file,
1272 # which we can then test for.
1273 contents = self.NEWFILE_CONTENTS * 300000
1274 d = self.PUT("/uri?format=mdmf",
1276 def _got_filecap(filecap):
1277 self.failUnless(filecap.startswith("URI:MDMF"))
1279 d.addCallback(_got_filecap)
1280 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1281 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1284 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1285 contents = self.NEWFILE_CONTENTS * 300000
1286 d = self.PUT("/uri?format=sdmf",
1288 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1289 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1292 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1293 contents = self.NEWFILE_CONTENTS * 300000
1294 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1295 400, "Bad Request", "Unknown format: foo",
1296 self.PUT, "/uri?format=foo",
1299 def test_PUT_NEWFILEURL_range_bad(self):
1300 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1301 target = self.public_url + "/foo/new.txt"
1302 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1303 "501 Not Implemented",
1304 "Content-Range in PUT not yet supported",
1305 # (and certainly not for immutable files)
1306 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1308 d.addCallback(lambda res:
1309 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1312 def test_PUT_NEWFILEURL_mutable(self):
1313 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1314 self.NEWFILE_CONTENTS)
1315 # TODO: we lose the response code, so we can't check this
1316 #self.failUnlessReallyEqual(responsecode, 201)
1317 def _check_uri(res):
1318 u = uri.from_string_mutable_filenode(res)
1319 self.failUnless(u.is_mutable())
1320 self.failIf(u.is_readonly())
1322 d.addCallback(_check_uri)
1323 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1324 d.addCallback(lambda res:
1325 self.failUnlessMutableChildContentsAre(self._foo_node,
1327 self.NEWFILE_CONTENTS))
1330 def test_PUT_NEWFILEURL_mutable_toobig(self):
1331 # It is okay to upload large mutable files, so we should be able
1333 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1334 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1337 def test_PUT_NEWFILEURL_replace(self):
1338 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1339 # TODO: we lose the response code, so we can't check this
1340 #self.failUnlessReallyEqual(responsecode, 200)
1341 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1342 d.addCallback(lambda res:
1343 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1344 self.NEWFILE_CONTENTS))
1347 def test_PUT_NEWFILEURL_bad_t(self):
1348 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1349 "PUT to a file: bad t=bogus",
1350 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1354 def test_PUT_NEWFILEURL_no_replace(self):
1355 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1356 self.NEWFILE_CONTENTS)
1357 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1359 "There was already a child by that name, and you asked me "
1360 "to not replace it")
1363 def test_PUT_NEWFILEURL_mkdirs(self):
1364 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1366 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1367 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1368 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1369 d.addCallback(lambda res:
1370 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1371 self.NEWFILE_CONTENTS))
1374 def test_PUT_NEWFILEURL_blocked(self):
1375 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1376 self.NEWFILE_CONTENTS)
1377 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1379 "Unable to create directory 'blockingfile': a file was in the way")
1382 def test_PUT_NEWFILEURL_emptyname(self):
1383 # an empty pathname component (i.e. a double-slash) is disallowed
1384 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1386 "The webapi does not allow empty pathname components",
1387 self.PUT, self.public_url + "/foo//new.txt", "")
1390 def test_DELETE_FILEURL(self):
1391 d = self.DELETE(self.public_url + "/foo/bar.txt")
1392 d.addCallback(lambda res:
1393 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1396 def test_DELETE_FILEURL_missing(self):
1397 d = self.DELETE(self.public_url + "/foo/missing")
1398 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1401 def test_DELETE_FILEURL_missing2(self):
1402 d = self.DELETE(self.public_url + "/missing/missing")
1403 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1406 def failUnlessHasBarDotTxtMetadata(self, res):
1407 data = simplejson.loads(res)
1408 self.failUnless(isinstance(data, list))
1409 self.failUnlessIn("metadata", data[1])
1410 self.failUnlessIn("tahoe", data[1]["metadata"])
1411 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1412 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1413 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1414 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1416 def test_GET_FILEURL_json(self):
1417 # twisted.web.http.parse_qs ignores any query args without an '=', so
1418 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1419 # instead. This may make it tricky to emulate the S3 interface
1421 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1423 self.failUnlessIsBarJSON(data)
1424 self.failUnlessHasBarDotTxtMetadata(data)
1426 d.addCallback(_check1)
1429 def test_GET_FILEURL_json_mutable_type(self):
1430 # The JSON should include format, which says whether the
1431 # file is SDMF or MDMF
1432 d = self.PUT("/uri?format=mdmf",
1433 self.NEWFILE_CONTENTS * 300000)
1434 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1435 def _got_json(json, version):
1436 data = simplejson.loads(json)
1437 assert "filenode" == data[0]
1439 assert isinstance(data, dict)
1441 self.failUnlessIn("format", data)
1442 self.failUnlessEqual(data["format"], version)
1444 d.addCallback(_got_json, "MDMF")
1445 # Now make an SDMF file and check that it is reported correctly.
1446 d.addCallback(lambda ignored:
1447 self.PUT("/uri?format=sdmf",
1448 self.NEWFILE_CONTENTS * 300000))
1449 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1450 d.addCallback(_got_json, "SDMF")
1453 def test_GET_FILEURL_json_mdmf(self):
1454 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1455 d.addCallback(self.failUnlessIsQuuxJSON)
1458 def test_GET_FILEURL_json_missing(self):
1459 d = self.GET(self.public_url + "/foo/missing?json")
1460 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1463 def test_GET_FILEURL_uri(self):
1464 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1466 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1467 d.addCallback(_check)
1468 d.addCallback(lambda res:
1469 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1471 # for now, for files, uris and readonly-uris are the same
1472 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1473 d.addCallback(_check2)
1476 def test_GET_FILEURL_badtype(self):
1477 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1480 self.public_url + "/foo/bar.txt?t=bogus")
1483 def test_CSS_FILE(self):
1484 d = self.GET("/tahoe.css", followRedirect=True)
1486 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1487 self.failUnless(CSS_STYLE.search(res), res)
1488 d.addCallback(_check)
1491 def test_GET_FILEURL_uri_missing(self):
1492 d = self.GET(self.public_url + "/foo/missing?t=uri")
1493 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1496 def _check_upload_and_mkdir_forms(self, html):
1497 # We should have a form to create a file, with radio buttons that allow
1498 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1499 self.failUnlessIn('name="t" value="upload"', html)
1500 self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1501 self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1502 self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1504 # We should also have the ability to create a mutable directory, with
1505 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1506 # or MDMF directory.
1507 self.failUnlessIn('name="t" value="mkdir"', html)
1508 self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1509 self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1511 self.failUnlessIn(FAVICON_MARKUP, html)
1513 def test_GET_DIRECTORY_html(self):
1514 d = self.GET(self.public_url + "/foo", followRedirect=True)
1516 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1517 self._check_upload_and_mkdir_forms(html)
1518 self.failUnlessIn("quux", html)
1519 d.addCallback(_check)
1522 def test_GET_DIRECTORY_html_filenode_encoding(self):
1523 d = self.GET(self.public_url + "/foo", followRedirect=True)
1525 # Check if encoded entries are there
1526 self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1527 + self._htmlname_escaped + '</a>', html)
1528 self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1529 self.failIfIn(self._htmlname_escaped_double, html)
1530 # Make sure that Nevow escaping actually works by checking for unsafe characters
1531 # and that '&' is escaped.
1533 self.failUnlessIn(entity, self._htmlname_raw)
1534 self.failIfIn(entity, self._htmlname_escaped)
1535 self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1536 self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1537 d.addCallback(_check)
1540 def test_GET_root_html(self):
1542 d.addCallback(self._check_upload_and_mkdir_forms)
1545 def test_GET_DIRURL(self):
1546 # the addSlash means we get a redirect here
1547 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1549 d = self.GET(self.public_url + "/foo", followRedirect=True)
1551 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1553 # the FILE reference points to a URI, but it should end in bar.txt
1554 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1555 (ROOT, urllib.quote(self._bar_txt_uri)))
1556 get_bar = "".join([r'<td>FILE</td>',
1558 r'<a href="%s">bar.txt</a>' % bar_url,
1560 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1562 self.failUnless(re.search(get_bar, res), res)
1563 for label in ['unlink', 'rename/move']:
1564 for line in res.split("\n"):
1565 # find the line that contains the relevant button for bar.txt
1566 if ("form action" in line and
1567 ('value="%s"' % (label,)) in line and
1568 'value="bar.txt"' in line):
1569 # the form target should use a relative URL
1570 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1571 self.failUnlessIn('action="%s"' % foo_url, line)
1572 # and the when_done= should too
1573 #done_url = urllib.quote(???)
1574 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1576 # 'unlink' needs to use POST because it directly has a side effect
1577 if label == 'unlink':
1578 self.failUnlessIn('method="post"', line)
1581 self.fail("unable to find '%s bar.txt' line" % (label,))
1583 # the DIR reference just points to a URI
1584 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1585 get_sub = ((r'<td>DIR</td>')
1586 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1587 self.failUnless(re.search(get_sub, res), res)
1588 d.addCallback(_check)
1590 # look at a readonly directory
1591 d.addCallback(lambda res:
1592 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1594 self.failUnlessIn("(read-only)", res)
1595 self.failIfIn("Upload a file", res)
1596 d.addCallback(_check2)
1598 # and at a directory that contains a readonly directory
1599 d.addCallback(lambda res:
1600 self.GET(self.public_url, followRedirect=True))
1602 self.failUnless(re.search('<td>DIR-RO</td>'
1603 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1604 d.addCallback(_check3)
1606 # and an empty directory
1607 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1609 self.failUnlessIn("directory is empty", res)
1610 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1611 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1612 d.addCallback(_check4)
1614 # and at a literal directory
1615 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1616 d.addCallback(lambda res:
1617 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1619 self.failUnlessIn('(immutable)', res)
1620 self.failUnless(re.search('<td>FILE</td>'
1621 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1622 d.addCallback(_check5)
1625 def test_GET_DIRURL_badtype(self):
1626 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1630 self.public_url + "/foo?t=bogus")
1633 def test_GET_DIRURL_json(self):
1634 d = self.GET(self.public_url + "/foo?t=json")
1635 d.addCallback(self.failUnlessIsFooJSON)
1638 def test_GET_DIRURL_json_format(self):
1639 d = self.PUT(self.public_url + \
1640 "/foo/sdmf.txt?format=sdmf",
1641 self.NEWFILE_CONTENTS * 300000)
1642 d.addCallback(lambda ignored:
1643 self.PUT(self.public_url + \
1644 "/foo/mdmf.txt?format=mdmf",
1645 self.NEWFILE_CONTENTS * 300000))
1646 # Now we have an MDMF and SDMF file in the directory. If we GET
1647 # its JSON, we should see their encodings.
1648 d.addCallback(lambda ignored:
1649 self.GET(self.public_url + "/foo?t=json"))
1650 def _got_json(json):
1651 data = simplejson.loads(json)
1652 assert data[0] == "dirnode"
1655 kids = data['children']
1657 mdmf_data = kids['mdmf.txt'][1]
1658 self.failUnlessIn("format", mdmf_data)
1659 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1661 sdmf_data = kids['sdmf.txt'][1]
1662 self.failUnlessIn("format", sdmf_data)
1663 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1664 d.addCallback(_got_json)
1668 def test_POST_DIRURL_manifest_no_ophandle(self):
1669 d = self.shouldFail2(error.Error,
1670 "test_POST_DIRURL_manifest_no_ophandle",
1672 "slow operation requires ophandle=",
1673 self.POST, self.public_url, t="start-manifest")
1676 def test_POST_DIRURL_manifest(self):
1677 d = defer.succeed(None)
1678 def getman(ignored, output):
1679 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1680 followRedirect=True)
1681 d.addCallback(self.wait_for_operation, "125")
1682 d.addCallback(self.get_operation_results, "125", output)
1684 d.addCallback(getman, None)
1685 def _got_html(manifest):
1686 self.failUnlessIn("Manifest of SI=", manifest)
1687 self.failUnlessIn("<td>sub</td>", manifest)
1688 self.failUnlessIn(self._sub_uri, manifest)
1689 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1690 self.failUnlessIn(FAVICON_MARKUP, manifest)
1691 d.addCallback(_got_html)
1693 # both t=status and unadorned GET should be identical
1694 d.addCallback(lambda res: self.GET("/operations/125"))
1695 d.addCallback(_got_html)
1697 d.addCallback(getman, "html")
1698 d.addCallback(_got_html)
1699 d.addCallback(getman, "text")
1700 def _got_text(manifest):
1701 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1702 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1703 d.addCallback(_got_text)
1704 d.addCallback(getman, "JSON")
1706 data = res["manifest"]
1708 for (path_list, cap) in data:
1709 got[tuple(path_list)] = cap
1710 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1711 self.failUnlessIn((u"sub", u"baz.txt"), got)
1712 self.failUnlessIn("finished", res)
1713 self.failUnlessIn("origin", res)
1714 self.failUnlessIn("storage-index", res)
1715 self.failUnlessIn("verifycaps", res)
1716 self.failUnlessIn("stats", res)
1717 d.addCallback(_got_json)
1720 def test_POST_DIRURL_deepsize_no_ophandle(self):
1721 d = self.shouldFail2(error.Error,
1722 "test_POST_DIRURL_deepsize_no_ophandle",
1724 "slow operation requires ophandle=",
1725 self.POST, self.public_url, t="start-deep-size")
1728 def test_POST_DIRURL_deepsize(self):
1729 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1730 followRedirect=True)
1731 d.addCallback(self.wait_for_operation, "126")
1732 d.addCallback(self.get_operation_results, "126", "json")
1733 def _got_json(data):
1734 self.failUnlessReallyEqual(data["finished"], True)
1736 self.failUnless(size > 1000)
1737 d.addCallback(_got_json)
1738 d.addCallback(self.get_operation_results, "126", "text")
1740 mo = re.search(r'^size: (\d+)$', res, re.M)
1741 self.failUnless(mo, res)
1742 size = int(mo.group(1))
1743 # with directories, the size varies.
1744 self.failUnless(size > 1000)
1745 d.addCallback(_got_text)
1748 def test_POST_DIRURL_deepstats_no_ophandle(self):
1749 d = self.shouldFail2(error.Error,
1750 "test_POST_DIRURL_deepstats_no_ophandle",
1752 "slow operation requires ophandle=",
1753 self.POST, self.public_url, t="start-deep-stats")
1756 def test_POST_DIRURL_deepstats(self):
1757 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1758 followRedirect=True)
1759 d.addCallback(self.wait_for_operation, "127")
1760 d.addCallback(self.get_operation_results, "127", "json")
1761 def _got_json(stats):
1762 expected = {"count-immutable-files": 4,
1763 "count-mutable-files": 2,
1764 "count-literal-files": 0,
1766 "count-directories": 3,
1767 "size-immutable-files": 76,
1768 "size-literal-files": 0,
1769 #"size-directories": 1912, # varies
1770 #"largest-directory": 1590,
1771 "largest-directory-children": 8,
1772 "largest-immutable-file": 19,
1774 for k,v in expected.iteritems():
1775 self.failUnlessReallyEqual(stats[k], v,
1776 "stats[%s] was %s, not %s" %
1778 self.failUnlessReallyEqual(stats["size-files-histogram"],
1780 d.addCallback(_got_json)
1783 def test_POST_DIRURL_stream_manifest(self):
1784 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1786 self.failUnless(res.endswith("\n"))
1787 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1788 self.failUnlessReallyEqual(len(units), 10)
1789 self.failUnlessEqual(units[-1]["type"], "stats")
1791 self.failUnlessEqual(first["path"], [])
1792 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1793 self.failUnlessEqual(first["type"], "directory")
1794 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1795 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1796 self.failIfEqual(baz["storage-index"], None)
1797 self.failIfEqual(baz["verifycap"], None)
1798 self.failIfEqual(baz["repaircap"], None)
1799 # XXX: Add quux and baz to this test.
1801 d.addCallback(_check)
1804 def test_GET_DIRURL_uri(self):
1805 d = self.GET(self.public_url + "/foo?t=uri")
1807 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1808 d.addCallback(_check)
1811 def test_GET_DIRURL_readonly_uri(self):
1812 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1814 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1815 d.addCallback(_check)
1818 def test_PUT_NEWDIRURL(self):
1819 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1820 d.addCallback(lambda res:
1821 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1822 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1823 d.addCallback(self.failUnlessNodeKeysAre, [])
1826 def test_PUT_NEWDIRURL_mdmf(self):
1827 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1828 d.addCallback(lambda res:
1829 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1830 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1831 d.addCallback(lambda node:
1832 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1835 def test_PUT_NEWDIRURL_sdmf(self):
1836 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1838 d.addCallback(lambda res:
1839 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1840 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1841 d.addCallback(lambda node:
1842 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1845 def test_PUT_NEWDIRURL_bad_format(self):
1846 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1847 400, "Bad Request", "Unknown format: foo",
1848 self.PUT, self.public_url +
1849 "/foo/newdir=?t=mkdir&format=foo", "")
1851 def test_POST_NEWDIRURL(self):
1852 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1853 d.addCallback(lambda res:
1854 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1855 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1856 d.addCallback(self.failUnlessNodeKeysAre, [])
1859 def test_POST_NEWDIRURL_mdmf(self):
1860 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1861 d.addCallback(lambda res:
1862 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1863 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1864 d.addCallback(lambda node:
1865 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1868 def test_POST_NEWDIRURL_sdmf(self):
1869 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1870 d.addCallback(lambda res:
1871 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1872 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1873 d.addCallback(lambda node:
1874 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1877 def test_POST_NEWDIRURL_bad_format(self):
1878 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1879 400, "Bad Request", "Unknown format: foo",
1880 self.POST2, self.public_url + \
1881 "/foo/newdir?t=mkdir&format=foo", "")
1883 def test_POST_NEWDIRURL_emptyname(self):
1884 # an empty pathname component (i.e. a double-slash) is disallowed
1885 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1887 "The webapi does not allow empty pathname components, i.e. a double slash",
1888 self.POST, self.public_url + "//?t=mkdir")
1891 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1892 (newkids, caps) = self._create_initial_children()
1893 query = "/foo/newdir?t=mkdir-with-children"
1894 if version == MDMF_VERSION:
1895 query += "&format=mdmf"
1896 elif version == SDMF_VERSION:
1897 query += "&format=sdmf"
1899 version = SDMF_VERSION # for later
1900 d = self.POST2(self.public_url + query,
1901 simplejson.dumps(newkids))
1903 n = self.s.create_node_from_uri(uri.strip())
1904 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1905 self.failUnlessEqual(n._node.get_version(), version)
1906 d2.addCallback(lambda ign:
1907 self.failUnlessROChildURIIs(n, u"child-imm",
1909 d2.addCallback(lambda ign:
1910 self.failUnlessRWChildURIIs(n, u"child-mutable",
1912 d2.addCallback(lambda ign:
1913 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1915 d2.addCallback(lambda ign:
1916 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1917 caps['unknown_rocap']))
1918 d2.addCallback(lambda ign:
1919 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1920 caps['unknown_rwcap']))
1921 d2.addCallback(lambda ign:
1922 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1923 caps['unknown_immcap']))
1924 d2.addCallback(lambda ign:
1925 self.failUnlessRWChildURIIs(n, u"dirchild",
1927 d2.addCallback(lambda ign:
1928 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1930 d2.addCallback(lambda ign:
1931 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1932 caps['emptydircap']))
1934 d.addCallback(_check)
1935 d.addCallback(lambda res:
1936 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1937 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1938 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1939 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1940 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1943 def test_POST_NEWDIRURL_initial_children(self):
1944 return self._do_POST_NEWDIRURL_initial_children_test()
1946 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1947 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1949 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1950 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1952 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1953 (newkids, caps) = self._create_initial_children()
1954 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1955 400, "Bad Request", "Unknown format: foo",
1956 self.POST2, self.public_url + \
1957 "/foo/newdir?t=mkdir-with-children&format=foo",
1958 simplejson.dumps(newkids))
1960 def test_POST_NEWDIRURL_immutable(self):
1961 (newkids, caps) = self._create_immutable_children()
1962 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1963 simplejson.dumps(newkids))
1965 n = self.s.create_node_from_uri(uri.strip())
1966 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1967 d2.addCallback(lambda ign:
1968 self.failUnlessROChildURIIs(n, u"child-imm",
1970 d2.addCallback(lambda ign:
1971 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1972 caps['unknown_immcap']))
1973 d2.addCallback(lambda ign:
1974 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1976 d2.addCallback(lambda ign:
1977 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1979 d2.addCallback(lambda ign:
1980 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1981 caps['emptydircap']))
1983 d.addCallback(_check)
1984 d.addCallback(lambda res:
1985 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1986 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1988 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1990 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1991 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1992 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1993 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1994 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1995 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1996 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1997 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1998 d.addErrback(self.explain_web_error)
2001 def test_POST_NEWDIRURL_immutable_bad(self):
2002 (newkids, caps) = self._create_initial_children()
2003 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2005 "needed to be immutable but was not",
2007 self.public_url + "/foo/newdir?t=mkdir-immutable",
2008 simplejson.dumps(newkids))
2011 def test_PUT_NEWDIRURL_exists(self):
2012 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2013 d.addCallback(lambda res:
2014 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2015 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2016 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2019 def test_PUT_NEWDIRURL_blocked(self):
2020 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2021 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2023 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2024 d.addCallback(lambda res:
2025 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2026 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2027 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2030 def test_PUT_NEWDIRURL_mkdirs(self):
2031 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2032 d.addCallback(lambda res:
2033 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2034 d.addCallback(lambda res:
2035 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2036 d.addCallback(lambda res:
2037 self._foo_node.get_child_at_path(u"subdir/newdir"))
2038 d.addCallback(self.failUnlessNodeKeysAre, [])
2041 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2042 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2043 d.addCallback(lambda ignored:
2044 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2045 d.addCallback(lambda ignored:
2046 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2047 d.addCallback(lambda ignored:
2048 self._foo_node.get_child_at_path(u"subdir"))
2049 def _got_subdir(subdir):
2050 # XXX: What we want?
2051 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2052 self.failUnlessNodeHasChild(subdir, u"newdir")
2053 return subdir.get_child_at_path(u"newdir")
2054 d.addCallback(_got_subdir)
2055 d.addCallback(lambda newdir:
2056 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2059 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2060 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2061 d.addCallback(lambda ignored:
2062 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2063 d.addCallback(lambda ignored:
2064 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2065 d.addCallback(lambda ignored:
2066 self._foo_node.get_child_at_path(u"subdir"))
2067 def _got_subdir(subdir):
2068 # XXX: What we want?
2069 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2070 self.failUnlessNodeHasChild(subdir, u"newdir")
2071 return subdir.get_child_at_path(u"newdir")
2072 d.addCallback(_got_subdir)
2073 d.addCallback(lambda newdir:
2074 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2077 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2078 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2079 400, "Bad Request", "Unknown format: foo",
2080 self.PUT, self.public_url + \
2081 "/foo/subdir/newdir?t=mkdir&format=foo",
2084 def test_DELETE_DIRURL(self):
2085 d = self.DELETE(self.public_url + "/foo")
2086 d.addCallback(lambda res:
2087 self.failIfNodeHasChild(self.public_root, u"foo"))
2090 def test_DELETE_DIRURL_missing(self):
2091 d = self.DELETE(self.public_url + "/foo/missing")
2092 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2093 d.addCallback(lambda res:
2094 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2097 def test_DELETE_DIRURL_missing2(self):
2098 d = self.DELETE(self.public_url + "/missing")
2099 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2102 def dump_root(self):
2104 w = webish.DirnodeWalkerMixin()
2105 def visitor(childpath, childnode, metadata):
2107 d = w.walk(self.public_root, visitor)
2110 def failUnlessNodeKeysAre(self, node, expected_keys):
2111 for k in expected_keys:
2112 assert isinstance(k, unicode)
2114 def _check(children):
2115 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2116 d.addCallback(_check)
2118 def failUnlessNodeHasChild(self, node, name):
2119 assert isinstance(name, unicode)
2121 def _check(children):
2122 self.failUnlessIn(name, children)
2123 d.addCallback(_check)
2125 def failIfNodeHasChild(self, node, name):
2126 assert isinstance(name, unicode)
2128 def _check(children):
2129 self.failIfIn(name, children)
2130 d.addCallback(_check)
2133 def failUnlessChildContentsAre(self, node, name, expected_contents):
2134 assert isinstance(name, unicode)
2135 d = node.get_child_at_path(name)
2136 d.addCallback(lambda node: download_to_data(node))
2137 def _check(contents):
2138 self.failUnlessReallyEqual(contents, expected_contents)
2139 d.addCallback(_check)
2142 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2143 assert isinstance(name, unicode)
2144 d = node.get_child_at_path(name)
2145 d.addCallback(lambda node: node.download_best_version())
2146 def _check(contents):
2147 self.failUnlessReallyEqual(contents, expected_contents)
2148 d.addCallback(_check)
2151 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2152 assert isinstance(name, unicode)
2153 d = node.get_child_at_path(name)
2155 self.failUnless(child.is_unknown() or not child.is_readonly())
2156 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2157 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2158 expected_ro_uri = self._make_readonly(expected_uri)
2160 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2161 d.addCallback(_check)
2164 def failUnlessROChildURIIs(self, node, name, expected_uri):
2165 assert isinstance(name, unicode)
2166 d = node.get_child_at_path(name)
2168 self.failUnless(child.is_unknown() or child.is_readonly())
2169 self.failUnlessReallyEqual(child.get_write_uri(), None)
2170 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2171 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2172 d.addCallback(_check)
2175 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2176 assert isinstance(name, unicode)
2177 d = node.get_child_at_path(name)
2179 self.failUnless(child.is_unknown() or not child.is_readonly())
2180 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2181 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2182 expected_ro_uri = self._make_readonly(got_uri)
2184 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2185 d.addCallback(_check)
2188 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2189 assert isinstance(name, unicode)
2190 d = node.get_child_at_path(name)
2192 self.failUnless(child.is_unknown() or child.is_readonly())
2193 self.failUnlessReallyEqual(child.get_write_uri(), None)
2194 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2195 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2196 d.addCallback(_check)
2199 def failUnlessCHKURIHasContents(self, got_uri, contents):
2200 self.failUnless(self.get_all_contents()[got_uri] == contents)
2202 def test_POST_upload(self):
2203 d = self.POST(self.public_url + "/foo", t="upload",
2204 file=("new.txt", self.NEWFILE_CONTENTS))
2206 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2207 d.addCallback(lambda res:
2208 self.failUnlessChildContentsAre(fn, u"new.txt",
2209 self.NEWFILE_CONTENTS))
2212 def test_POST_upload_unicode(self):
2213 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2214 d = self.POST(self.public_url + "/foo", t="upload",
2215 file=(filename, self.NEWFILE_CONTENTS))
2217 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2218 d.addCallback(lambda res:
2219 self.failUnlessChildContentsAre(fn, filename,
2220 self.NEWFILE_CONTENTS))
2221 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2222 d.addCallback(lambda res: self.GET(target_url))
2223 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2224 self.NEWFILE_CONTENTS,
2228 def test_POST_upload_unicode_named(self):
2229 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2230 d = self.POST(self.public_url + "/foo", t="upload",
2232 file=("overridden", self.NEWFILE_CONTENTS))
2234 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2235 d.addCallback(lambda res:
2236 self.failUnlessChildContentsAre(fn, filename,
2237 self.NEWFILE_CONTENTS))
2238 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2239 d.addCallback(lambda res: self.GET(target_url))
2240 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2241 self.NEWFILE_CONTENTS,
2245 def test_POST_upload_no_link(self):
2246 d = self.POST("/uri", t="upload",
2247 file=("new.txt", self.NEWFILE_CONTENTS))
2248 def _check_upload_results(page):
2249 # this should be a page which describes the results of the upload
2250 # that just finished.
2251 self.failUnlessIn("Upload Results:", page)
2252 self.failUnlessIn("URI:", page)
2253 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2254 mo = uri_re.search(page)
2255 self.failUnless(mo, page)
2256 new_uri = mo.group(1)
2258 d.addCallback(_check_upload_results)
2259 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2262 def test_POST_upload_no_link_whendone(self):
2263 d = self.POST("/uri", t="upload", when_done="/",
2264 file=("new.txt", self.NEWFILE_CONTENTS))
2265 d.addBoth(self.shouldRedirect, "/")
2268 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2269 d = defer.maybeDeferred(callable, *args, **kwargs)
2271 if isinstance(res, failure.Failure):
2272 res.trap(error.PageRedirect)
2273 statuscode = res.value.status
2274 target = res.value.location
2275 return checker(statuscode, target)
2276 self.fail("%s: callable was supposed to redirect, not return '%s'"
2281 def test_POST_upload_no_link_whendone_results(self):
2282 def check(statuscode, target):
2283 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2284 self.failUnless(target.startswith(self.webish_url), target)
2285 return client.getPage(target, method="GET")
2286 # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2287 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2289 self.POST, "/uri", t="upload",
2290 when_done="/%75ri/%(uri)s",
2291 file=("new.txt", self.NEWFILE_CONTENTS))
2292 d.addCallback(lambda res:
2293 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2296 def test_POST_upload_no_link_mutable(self):
2297 d = self.POST("/uri", t="upload", mutable="true",
2298 file=("new.txt", self.NEWFILE_CONTENTS))
2299 def _check(filecap):
2300 filecap = filecap.strip()
2301 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2302 self.filecap = filecap
2303 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2304 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2305 n = self.s.create_node_from_uri(filecap)
2306 return n.download_best_version()
2307 d.addCallback(_check)
2309 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2310 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2311 d.addCallback(_check2)
2313 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2314 return self.GET("/file/%s" % urllib.quote(self.filecap))
2315 d.addCallback(_check3)
2317 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2318 d.addCallback(_check4)
2321 def test_POST_upload_no_link_mutable_toobig(self):
2322 # The SDMF size limit is no longer in place, so we should be
2323 # able to upload mutable files that are as large as we want them
2325 d = self.POST("/uri", t="upload", mutable="true",
2326 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2330 def test_POST_upload_format_unlinked(self):
2331 def _check_upload_unlinked(ign, format, uri_prefix):
2332 filename = format + ".txt"
2333 d = self.POST("/uri?t=upload&format=" + format,
2334 file=(filename, self.NEWFILE_CONTENTS * 300000))
2335 def _got_results(results):
2336 if format.upper() in ("SDMF", "MDMF"):
2337 # webapi.rst says this returns a filecap
2340 # for immutable, it returns an "upload results page", and
2341 # the filecap is buried inside
2342 line = [l for l in results.split("\n") if "URI: " in l][0]
2343 mo = re.search(r'<span>([^<]+)</span>', line)
2344 filecap = mo.group(1)
2345 self.failUnless(filecap.startswith(uri_prefix),
2346 (uri_prefix, filecap))
2347 return self.GET("/uri/%s?t=json" % filecap)
2348 d.addCallback(_got_results)
2349 def _got_json(json):
2350 data = simplejson.loads(json)
2352 self.failUnlessIn("format", data)
2353 self.failUnlessEqual(data["format"], format.upper())
2354 d.addCallback(_got_json)
2356 d = defer.succeed(None)
2357 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2358 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2359 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2360 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2363 def test_POST_upload_bad_format_unlinked(self):
2364 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2365 400, "Bad Request", "Unknown format: foo",
2367 "/uri?t=upload&format=foo",
2368 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2370 def test_POST_upload_format(self):
2371 def _check_upload(ign, format, uri_prefix, fn=None):
2372 filename = format + ".txt"
2373 d = self.POST(self.public_url +
2374 "/foo?t=upload&format=" + format,
2375 file=(filename, self.NEWFILE_CONTENTS * 300000))
2376 def _got_filecap(filecap):
2378 filenameu = unicode(filename)
2379 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2380 self.failUnless(filecap.startswith(uri_prefix))
2381 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2382 d.addCallback(_got_filecap)
2383 def _got_json(json):
2384 data = simplejson.loads(json)
2386 self.failUnlessIn("format", data)
2387 self.failUnlessEqual(data["format"], format.upper())
2388 d.addCallback(_got_json)
2391 d = defer.succeed(None)
2392 d.addCallback(_check_upload, "chk", "URI:CHK")
2393 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2394 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2395 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2398 def test_POST_upload_bad_format(self):
2399 return self.shouldHTTPError("POST_upload_bad_format",
2400 400, "Bad Request", "Unknown format: foo",
2401 self.POST, self.public_url + \
2402 "/foo?t=upload&format=foo",
2403 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2405 def test_POST_upload_mutable(self):
2406 # this creates a mutable file
2407 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2408 file=("new.txt", self.NEWFILE_CONTENTS))
2410 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2411 d.addCallback(lambda res:
2412 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2413 self.NEWFILE_CONTENTS))
2414 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2416 self.failUnless(IMutableFileNode.providedBy(newnode))
2417 self.failUnless(newnode.is_mutable())
2418 self.failIf(newnode.is_readonly())
2419 self._mutable_node = newnode
2420 self._mutable_uri = newnode.get_uri()
2423 # now upload it again and make sure that the URI doesn't change
2424 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2425 d.addCallback(lambda res:
2426 self.POST(self.public_url + "/foo", t="upload",
2428 file=("new.txt", NEWER_CONTENTS)))
2429 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2430 d.addCallback(lambda res:
2431 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2433 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2435 self.failUnless(IMutableFileNode.providedBy(newnode))
2436 self.failUnless(newnode.is_mutable())
2437 self.failIf(newnode.is_readonly())
2438 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2439 d.addCallback(_got2)
2441 # upload a second time, using PUT instead of POST
2442 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2443 d.addCallback(lambda res:
2444 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2445 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2446 d.addCallback(lambda res:
2447 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2450 # finally list the directory, since mutable files are displayed
2451 # slightly differently
2453 d.addCallback(lambda res:
2454 self.GET(self.public_url + "/foo/",
2455 followRedirect=True))
2456 def _check_page(res):
2457 # TODO: assert more about the contents
2458 self.failUnlessIn("SSK", res)
2460 d.addCallback(_check_page)
2462 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2464 self.failUnless(IMutableFileNode.providedBy(newnode))
2465 self.failUnless(newnode.is_mutable())
2466 self.failIf(newnode.is_readonly())
2467 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2468 d.addCallback(_got3)
2470 # look at the JSON form of the enclosing directory
2471 d.addCallback(lambda res:
2472 self.GET(self.public_url + "/foo/?t=json",
2473 followRedirect=True))
2474 def _check_page_json(res):
2475 parsed = simplejson.loads(res)
2476 self.failUnlessEqual(parsed[0], "dirnode")
2477 children = dict( [(unicode(name),value)
2479 in parsed[1]["children"].iteritems()] )
2480 self.failUnlessIn(u"new.txt", children)
2481 new_json = children[u"new.txt"]
2482 self.failUnlessEqual(new_json[0], "filenode")
2483 self.failUnless(new_json[1]["mutable"])
2484 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2485 ro_uri = self._mutable_node.get_readonly().to_string()
2486 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2487 d.addCallback(_check_page_json)
2489 # and the JSON form of the file
2490 d.addCallback(lambda res:
2491 self.GET(self.public_url + "/foo/new.txt?t=json"))
2492 def _check_file_json(res):
2493 parsed = simplejson.loads(res)
2494 self.failUnlessEqual(parsed[0], "filenode")
2495 self.failUnless(parsed[1]["mutable"])
2496 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2497 ro_uri = self._mutable_node.get_readonly().to_string()
2498 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2499 d.addCallback(_check_file_json)
2501 # and look at t=uri and t=readonly-uri
2502 d.addCallback(lambda res:
2503 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2504 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2505 d.addCallback(lambda res:
2506 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2507 def _check_ro_uri(res):
2508 ro_uri = self._mutable_node.get_readonly().to_string()
2509 self.failUnlessReallyEqual(res, ro_uri)
2510 d.addCallback(_check_ro_uri)
2512 # make sure we can get to it from /uri/URI
2513 d.addCallback(lambda res:
2514 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2515 d.addCallback(lambda res:
2516 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2518 # and that HEAD computes the size correctly
2519 d.addCallback(lambda res:
2520 self.HEAD(self.public_url + "/foo/new.txt",
2521 return_response=True))
2522 def _got_headers((res, status, headers)):
2523 self.failUnlessReallyEqual(res, "")
2524 self.failUnlessReallyEqual(headers["content-length"][0],
2525 str(len(NEW2_CONTENTS)))
2526 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2527 d.addCallback(_got_headers)
2529 # make sure that outdated size limits aren't enforced anymore.
2530 d.addCallback(lambda ignored:
2531 self.POST(self.public_url + "/foo", t="upload",
2534 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2535 d.addErrback(self.dump_error)
2538 def test_POST_upload_mutable_toobig(self):
2539 # SDMF had a size limti that was removed a while ago. MDMF has
2540 # never had a size limit. Test to make sure that we do not
2541 # encounter errors when trying to upload large mutable files,
2542 # since there should be no coded prohibitions regarding large
2544 d = self.POST(self.public_url + "/foo",
2545 t="upload", mutable="true",
2546 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2549 def dump_error(self, f):
2550 # if the web server returns an error code (like 400 Bad Request),
2551 # web.client.getPage puts the HTTP response body into the .response
2552 # attribute of the exception object that it gives back. It does not
2553 # appear in the Failure's repr(), so the ERROR that trial displays
2554 # will be rather terse and unhelpful. addErrback this method to the
2555 # end of your chain to get more information out of these errors.
2556 if f.check(error.Error):
2557 print "web.error.Error:"
2559 print f.value.response
2562 def test_POST_upload_replace(self):
2563 d = self.POST(self.public_url + "/foo", t="upload",
2564 file=("bar.txt", self.NEWFILE_CONTENTS))
2566 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2567 d.addCallback(lambda res:
2568 self.failUnlessChildContentsAre(fn, u"bar.txt",
2569 self.NEWFILE_CONTENTS))
2572 def test_POST_upload_no_replace_ok(self):
2573 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2574 file=("new.txt", self.NEWFILE_CONTENTS))
2575 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2576 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2577 self.NEWFILE_CONTENTS))
2580 def test_POST_upload_no_replace_queryarg(self):
2581 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2582 file=("bar.txt", self.NEWFILE_CONTENTS))
2583 d.addBoth(self.shouldFail, error.Error,
2584 "POST_upload_no_replace_queryarg",
2586 "There was already a child by that name, and you asked me "
2587 "to not replace it")
2588 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2589 d.addCallback(self.failUnlessIsBarDotTxt)
2592 def test_POST_upload_no_replace_field(self):
2593 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2594 file=("bar.txt", self.NEWFILE_CONTENTS))
2595 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2597 "There was already a child by that name, and you asked me "
2598 "to not replace it")
2599 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2600 d.addCallback(self.failUnlessIsBarDotTxt)
2603 def test_POST_upload_whendone(self):
2604 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2605 file=("new.txt", self.NEWFILE_CONTENTS))
2606 d.addBoth(self.shouldRedirect, "/THERE")
2608 d.addCallback(lambda res:
2609 self.failUnlessChildContentsAre(fn, u"new.txt",
2610 self.NEWFILE_CONTENTS))
2613 def test_POST_upload_named(self):
2615 d = self.POST(self.public_url + "/foo", t="upload",
2616 name="new.txt", file=self.NEWFILE_CONTENTS)
2617 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2618 d.addCallback(lambda res:
2619 self.failUnlessChildContentsAre(fn, u"new.txt",
2620 self.NEWFILE_CONTENTS))
2623 def test_POST_upload_named_badfilename(self):
2624 d = self.POST(self.public_url + "/foo", t="upload",
2625 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2626 d.addBoth(self.shouldFail, error.Error,
2627 "test_POST_upload_named_badfilename",
2629 "name= may not contain a slash",
2631 # make sure that nothing was added
2632 d.addCallback(lambda res:
2633 self.failUnlessNodeKeysAre(self._foo_node,
2634 [self._htmlname_unicode,
2635 u"bar.txt", u"baz.txt", u"blockingfile",
2636 u"empty", u"n\u00fc.txt", u"quux.txt",
2640 def test_POST_FILEURL_check(self):
2641 bar_url = self.public_url + "/foo/bar.txt"
2642 d = self.POST(bar_url, t="check")
2644 self.failUnlessIn("Healthy :", res)
2645 d.addCallback(_check)
2646 redir_url = "http://allmydata.org/TARGET"
2647 def _check2(statuscode, target):
2648 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2649 self.failUnlessReallyEqual(target, redir_url)
2650 d.addCallback(lambda res:
2651 self.shouldRedirect2("test_POST_FILEURL_check",
2655 when_done=redir_url))
2656 d.addCallback(lambda res:
2657 self.POST(bar_url, t="check", return_to=redir_url))
2659 self.failUnlessIn("Healthy :", res)
2660 self.failUnlessIn("Return to file", res)
2661 self.failUnlessIn(redir_url, res)
2662 d.addCallback(_check3)
2664 d.addCallback(lambda res:
2665 self.POST(bar_url, t="check", output="JSON"))
2666 def _check_json(res):
2667 data = simplejson.loads(res)
2668 self.failUnlessIn("storage-index", data)
2669 self.failUnless(data["results"]["healthy"])
2670 d.addCallback(_check_json)
2674 def test_POST_FILEURL_check_and_repair(self):
2675 bar_url = self.public_url + "/foo/bar.txt"
2676 d = self.POST(bar_url, t="check", repair="true")
2678 self.failUnlessIn("Healthy :", res)
2679 d.addCallback(_check)
2680 redir_url = "http://allmydata.org/TARGET"
2681 def _check2(statuscode, target):
2682 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2683 self.failUnlessReallyEqual(target, redir_url)
2684 d.addCallback(lambda res:
2685 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2688 t="check", repair="true",
2689 when_done=redir_url))
2690 d.addCallback(lambda res:
2691 self.POST(bar_url, t="check", return_to=redir_url))
2693 self.failUnlessIn("Healthy :", res)
2694 self.failUnlessIn("Return to file", res)
2695 self.failUnlessIn(redir_url, res)
2696 d.addCallback(_check3)
2699 def test_POST_DIRURL_check(self):
2700 foo_url = self.public_url + "/foo/"
2701 d = self.POST(foo_url, t="check")
2703 self.failUnlessIn("Healthy :", res)
2704 d.addCallback(_check)
2705 redir_url = "http://allmydata.org/TARGET"
2706 def _check2(statuscode, target):
2707 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2708 self.failUnlessReallyEqual(target, redir_url)
2709 d.addCallback(lambda res:
2710 self.shouldRedirect2("test_POST_DIRURL_check",
2714 when_done=redir_url))
2715 d.addCallback(lambda res:
2716 self.POST(foo_url, t="check", return_to=redir_url))
2718 self.failUnlessIn("Healthy :", res)
2719 self.failUnlessIn("Return to file/directory", res)
2720 self.failUnlessIn(redir_url, res)
2721 d.addCallback(_check3)
2723 d.addCallback(lambda res:
2724 self.POST(foo_url, t="check", output="JSON"))
2725 def _check_json(res):
2726 data = simplejson.loads(res)
2727 self.failUnlessIn("storage-index", data)
2728 self.failUnless(data["results"]["healthy"])
2729 d.addCallback(_check_json)
2733 def test_POST_DIRURL_check_and_repair(self):
2734 foo_url = self.public_url + "/foo/"
2735 d = self.POST(foo_url, t="check", repair="true")
2737 self.failUnlessIn("Healthy :", res)
2738 d.addCallback(_check)
2739 redir_url = "http://allmydata.org/TARGET"
2740 def _check2(statuscode, target):
2741 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2742 self.failUnlessReallyEqual(target, redir_url)
2743 d.addCallback(lambda res:
2744 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2747 t="check", repair="true",
2748 when_done=redir_url))
2749 d.addCallback(lambda res:
2750 self.POST(foo_url, t="check", return_to=redir_url))
2752 self.failUnlessIn("Healthy :", res)
2753 self.failUnlessIn("Return to file/directory", res)
2754 self.failUnlessIn(redir_url, res)
2755 d.addCallback(_check3)
2758 def test_POST_FILEURL_mdmf_check(self):
2759 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2760 d = self.POST(quux_url, t="check")
2762 self.failUnlessIn("Healthy", res)
2763 d.addCallback(_check)
2764 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2765 d.addCallback(lambda ignored:
2766 self.POST(quux_extension_url, t="check"))
2767 d.addCallback(_check)
2770 def test_POST_FILEURL_mdmf_check_and_repair(self):
2771 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2772 d = self.POST(quux_url, t="check", repair="true")
2774 self.failUnlessIn("Healthy", res)
2775 d.addCallback(_check)
2776 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2777 d.addCallback(lambda ignored:
2778 self.POST(quux_extension_url, t="check", repair="true"))
2779 d.addCallback(_check)
2782 def wait_for_operation(self, ignored, ophandle):
2783 url = "/operations/" + ophandle
2784 url += "?t=status&output=JSON"
2787 data = simplejson.loads(res)
2788 if not data["finished"]:
2789 d = self.stall(delay=1.0)
2790 d.addCallback(self.wait_for_operation, ophandle)
2796 def get_operation_results(self, ignored, ophandle, output=None):
2797 url = "/operations/" + ophandle
2800 url += "&output=" + output
2803 if output and output.lower() == "json":
2804 return simplejson.loads(res)
2809 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2810 d = self.shouldFail2(error.Error,
2811 "test_POST_DIRURL_deepcheck_no_ophandle",
2813 "slow operation requires ophandle=",
2814 self.POST, self.public_url, t="start-deep-check")
2817 def test_POST_DIRURL_deepcheck(self):
2818 def _check_redirect(statuscode, target):
2819 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2820 self.failUnless(target.endswith("/operations/123"))
2821 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2822 self.POST, self.public_url,
2823 t="start-deep-check", ophandle="123")
2824 d.addCallback(self.wait_for_operation, "123")
2825 def _check_json(data):
2826 self.failUnlessReallyEqual(data["finished"], True)
2827 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2828 self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2829 d.addCallback(_check_json)
2830 d.addCallback(self.get_operation_results, "123", "html")
2831 def _check_html(res):
2832 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2833 self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2834 self.failUnlessIn(FAVICON_MARKUP, res)
2835 d.addCallback(_check_html)
2837 d.addCallback(lambda res:
2838 self.GET("/operations/123/"))
2839 d.addCallback(_check_html) # should be the same as without the slash
2841 d.addCallback(lambda res:
2842 self.shouldFail2(error.Error, "one", "404 Not Found",
2843 "No detailed results for SI bogus",
2844 self.GET, "/operations/123/bogus"))
2846 foo_si = self._foo_node.get_storage_index()
2847 foo_si_s = base32.b2a(foo_si)
2848 d.addCallback(lambda res:
2849 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2850 def _check_foo_json(res):
2851 data = simplejson.loads(res)
2852 self.failUnlessEqual(data["storage-index"], foo_si_s)
2853 self.failUnless(data["results"]["healthy"])
2854 d.addCallback(_check_foo_json)
2857 def test_POST_DIRURL_deepcheck_and_repair(self):
2858 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2859 ophandle="124", output="json", followRedirect=True)
2860 d.addCallback(self.wait_for_operation, "124")
2861 def _check_json(data):
2862 self.failUnlessReallyEqual(data["finished"], True)
2863 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2864 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2865 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2866 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2867 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2868 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2869 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2870 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2871 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2872 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2873 d.addCallback(_check_json)
2874 d.addCallback(self.get_operation_results, "124", "html")
2875 def _check_html(res):
2876 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2878 self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2879 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2880 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2882 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2883 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2884 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2886 self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2887 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2888 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2890 self.failUnlessIn(FAVICON_MARKUP, res)
2891 d.addCallback(_check_html)
2894 def test_POST_FILEURL_bad_t(self):
2895 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2896 "POST to file: bad t=bogus",
2897 self.POST, self.public_url + "/foo/bar.txt",
2901 def test_POST_mkdir(self): # return value?
2902 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2903 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2904 d.addCallback(self.failUnlessNodeKeysAre, [])
2907 def test_POST_mkdir_mdmf(self):
2908 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2909 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2910 d.addCallback(lambda node:
2911 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2914 def test_POST_mkdir_sdmf(self):
2915 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2916 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917 d.addCallback(lambda node:
2918 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2921 def test_POST_mkdir_bad_format(self):
2922 return self.shouldHTTPError("POST_mkdir_bad_format",
2923 400, "Bad Request", "Unknown format: foo",
2924 self.POST, self.public_url +
2925 "/foo?t=mkdir&name=newdir&format=foo")
2927 def test_POST_mkdir_initial_children(self):
2928 (newkids, caps) = self._create_initial_children()
2929 d = self.POST2(self.public_url +
2930 "/foo?t=mkdir-with-children&name=newdir",
2931 simplejson.dumps(newkids))
2932 d.addCallback(lambda res:
2933 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2934 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2935 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2936 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2940 def test_POST_mkdir_initial_children_mdmf(self):
2941 (newkids, caps) = self._create_initial_children()
2942 d = self.POST2(self.public_url +
2943 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2944 simplejson.dumps(newkids))
2945 d.addCallback(lambda res:
2946 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2947 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948 d.addCallback(lambda node:
2949 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2950 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2951 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2956 def test_POST_mkdir_initial_children_sdmf(self):
2957 (newkids, caps) = self._create_initial_children()
2958 d = self.POST2(self.public_url +
2959 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2960 simplejson.dumps(newkids))
2961 d.addCallback(lambda res:
2962 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2963 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964 d.addCallback(lambda node:
2965 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2966 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2967 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2971 def test_POST_mkdir_initial_children_bad_format(self):
2972 (newkids, caps) = self._create_initial_children()
2973 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2974 400, "Bad Request", "Unknown format: foo",
2975 self.POST, self.public_url + \
2976 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2977 simplejson.dumps(newkids))
2979 def test_POST_mkdir_immutable(self):
2980 (newkids, caps) = self._create_immutable_children()
2981 d = self.POST2(self.public_url +
2982 "/foo?t=mkdir-immutable&name=newdir",
2983 simplejson.dumps(newkids))
2984 d.addCallback(lambda res:
2985 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2986 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2987 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2988 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2989 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2990 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2991 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2992 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2993 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2994 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2995 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2996 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2997 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3000 def test_POST_mkdir_immutable_bad(self):
3001 (newkids, caps) = self._create_initial_children()
3002 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3004 "needed to be immutable but was not",
3007 "/foo?t=mkdir-immutable&name=newdir",
3008 simplejson.dumps(newkids))
3011 def test_POST_mkdir_2(self):
3012 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3013 d.addCallback(lambda res:
3014 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3015 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3016 d.addCallback(self.failUnlessNodeKeysAre, [])
3019 def test_POST_mkdirs_2(self):
3020 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3021 d.addCallback(lambda res:
3022 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3023 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3024 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3025 d.addCallback(self.failUnlessNodeKeysAre, [])
3028 def test_POST_mkdir_no_parentdir_noredirect(self):
3029 d = self.POST("/uri?t=mkdir")
3030 def _after_mkdir(res):
3031 uri.DirectoryURI.init_from_string(res)
3032 d.addCallback(_after_mkdir)
3035 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3036 d = self.POST("/uri?t=mkdir&format=mdmf")
3037 def _after_mkdir(res):
3038 u = uri.from_string(res)
3039 # Check that this is an MDMF writecap
3040 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3041 d.addCallback(_after_mkdir)
3044 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3045 d = self.POST("/uri?t=mkdir&format=sdmf")
3046 def _after_mkdir(res):
3047 u = uri.from_string(res)
3048 self.failUnlessIsInstance(u, uri.DirectoryURI)
3049 d.addCallback(_after_mkdir)
3052 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3053 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3054 400, "Bad Request", "Unknown format: foo",
3055 self.POST, self.public_url +
3056 "/uri?t=mkdir&format=foo")
3058 def test_POST_mkdir_no_parentdir_noredirect2(self):
3059 # make sure form-based arguments (as on the welcome page) still work
3060 d = self.POST("/uri", t="mkdir")
3061 def _after_mkdir(res):
3062 uri.DirectoryURI.init_from_string(res)
3063 d.addCallback(_after_mkdir)
3064 d.addErrback(self.explain_web_error)
3067 def test_POST_mkdir_no_parentdir_redirect(self):
3068 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3069 d.addBoth(self.shouldRedirect, None, statuscode='303')
3070 def _check_target(target):
3071 target = urllib.unquote(target)
3072 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3073 d.addCallback(_check_target)
3076 def test_POST_mkdir_no_parentdir_redirect2(self):
3077 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3078 d.addBoth(self.shouldRedirect, None, statuscode='303')
3079 def _check_target(target):
3080 target = urllib.unquote(target)
3081 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3082 d.addCallback(_check_target)
3083 d.addErrback(self.explain_web_error)
3086 def _make_readonly(self, u):
3087 ro_uri = uri.from_string(u).get_readonly()
3090 return ro_uri.to_string()
3092 def _create_initial_children(self):
3093 contents, n, filecap1 = self.makefile(12)
3094 md1 = {"metakey1": "metavalue1"}
3095 filecap2 = make_mutable_file_uri()
3096 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3097 filecap3 = node3.get_readonly_uri()
3098 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3099 dircap = DirectoryNode(node4, None, None).get_uri()
3100 mdmfcap = make_mutable_file_uri(mdmf=True)
3101 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3102 emptydircap = "URI:DIR2-LIT:"
3103 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3104 "ro_uri": self._make_readonly(filecap1),
3105 "metadata": md1, }],
3106 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3107 "ro_uri": self._make_readonly(filecap2)}],
3108 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3109 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3110 "ro_uri": unknown_rocap}],
3111 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3112 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3113 u"dirchild": ["dirnode", {"rw_uri": dircap,
3114 "ro_uri": self._make_readonly(dircap)}],
3115 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3116 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3117 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3118 "ro_uri": self._make_readonly(mdmfcap)}],
3120 return newkids, {'filecap1': filecap1,
3121 'filecap2': filecap2,
3122 'filecap3': filecap3,
3123 'unknown_rwcap': unknown_rwcap,
3124 'unknown_rocap': unknown_rocap,
3125 'unknown_immcap': unknown_immcap,
3127 'litdircap': litdircap,
3128 'emptydircap': emptydircap,
3131 def _create_immutable_children(self):
3132 contents, n, filecap1 = self.makefile(12)
3133 md1 = {"metakey1": "metavalue1"}
3134 tnode = create_chk_filenode("immutable directory contents\n"*10,
3135 self.get_all_contents())
3136 dnode = DirectoryNode(tnode, None, None)
3137 assert not dnode.is_mutable()
3138 immdircap = dnode.get_uri()
3139 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3140 emptydircap = "URI:DIR2-LIT:"
3141 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3142 "metadata": md1, }],
3143 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3144 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3145 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3146 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3148 return newkids, {'filecap1': filecap1,
3149 'unknown_immcap': unknown_immcap,
3150 'immdircap': immdircap,
3151 'litdircap': litdircap,
3152 'emptydircap': emptydircap}
3154 def test_POST_mkdir_no_parentdir_initial_children(self):
3155 (newkids, caps) = self._create_initial_children()
3156 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3157 def _after_mkdir(res):
3158 self.failUnless(res.startswith("URI:DIR"), res)
3159 n = self.s.create_node_from_uri(res)
3160 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3161 d2.addCallback(lambda ign:
3162 self.failUnlessROChildURIIs(n, u"child-imm",
3164 d2.addCallback(lambda ign:
3165 self.failUnlessRWChildURIIs(n, u"child-mutable",
3167 d2.addCallback(lambda ign:
3168 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3170 d2.addCallback(lambda ign:
3171 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3172 caps['unknown_rwcap']))
3173 d2.addCallback(lambda ign:
3174 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3175 caps['unknown_rocap']))
3176 d2.addCallback(lambda ign:
3177 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3178 caps['unknown_immcap']))
3179 d2.addCallback(lambda ign:
3180 self.failUnlessRWChildURIIs(n, u"dirchild",
3183 d.addCallback(_after_mkdir)
3186 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3187 # the regular /uri?t=mkdir operation is specified to ignore its body.
3188 # Only t=mkdir-with-children pays attention to it.
3189 (newkids, caps) = self._create_initial_children()
3190 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3192 "t=mkdir does not accept children=, "
3193 "try t=mkdir-with-children instead",
3194 self.POST2, "/uri?t=mkdir", # without children
3195 simplejson.dumps(newkids))
3198 def test_POST_noparent_bad(self):
3199 d = self.shouldHTTPError("POST_noparent_bad",
3201 "/uri accepts only PUT, PUT?t=mkdir, "
3202 "POST?t=upload, and POST?t=mkdir",
3203 self.POST, "/uri?t=bogus")
3206 def test_POST_mkdir_no_parentdir_immutable(self):
3207 (newkids, caps) = self._create_immutable_children()
3208 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3209 def _after_mkdir(res):
3210 self.failUnless(res.startswith("URI:DIR"), res)
3211 n = self.s.create_node_from_uri(res)
3212 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3213 d2.addCallback(lambda ign:
3214 self.failUnlessROChildURIIs(n, u"child-imm",
3216 d2.addCallback(lambda ign:
3217 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3218 caps['unknown_immcap']))
3219 d2.addCallback(lambda ign:
3220 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3222 d2.addCallback(lambda ign:
3223 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3225 d2.addCallback(lambda ign:
3226 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3227 caps['emptydircap']))
3229 d.addCallback(_after_mkdir)
3232 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3233 (newkids, caps) = self._create_initial_children()
3234 d = self.shouldFail2(error.Error,
3235 "test_POST_mkdir_no_parentdir_immutable_bad",
3237 "needed to be immutable but was not",
3239 "/uri?t=mkdir-immutable",
3240 simplejson.dumps(newkids))
3243 def test_welcome_page_mkdir_button(self):
3244 # Fetch the welcome page.
3246 def _after_get_welcome_page(res):
3247 MKDIR_BUTTON_RE = re.compile(
3248 '<form action="([^"]*)" method="post".*'
3249 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3250 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3251 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3252 html = res.replace('\n', ' ')
3253 mo = MKDIR_BUTTON_RE.search(html)
3254 self.failUnless(mo, html)
3255 formaction = mo.group(1)
3257 formaname = mo.group(3)
3258 formavalue = mo.group(4)
3259 return (formaction, formt, formaname, formavalue)
3260 d.addCallback(_after_get_welcome_page)
3261 def _after_parse_form(res):
3262 (formaction, formt, formaname, formavalue) = res
3263 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3264 d.addCallback(_after_parse_form)
3265 d.addBoth(self.shouldRedirect, None, statuscode='303')
3268 def test_POST_mkdir_replace(self): # return value?
3269 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3270 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3271 d.addCallback(self.failUnlessNodeKeysAre, [])
3274 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3275 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3276 d.addBoth(self.shouldFail, error.Error,
3277 "POST_mkdir_no_replace_queryarg",
3279 "There was already a child by that name, and you asked me "
3280 "to not replace it")
3281 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3282 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3285 def test_POST_mkdir_no_replace_field(self): # return value?
3286 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3288 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3290 "There was already a child by that name, and you asked me "
3291 "to not replace it")
3292 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3293 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3296 def test_POST_mkdir_whendone_field(self):
3297 d = self.POST(self.public_url + "/foo",
3298 t="mkdir", name="newdir", when_done="/THERE")
3299 d.addBoth(self.shouldRedirect, "/THERE")
3300 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3301 d.addCallback(self.failUnlessNodeKeysAre, [])
3304 def test_POST_mkdir_whendone_queryarg(self):
3305 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3306 t="mkdir", name="newdir")
3307 d.addBoth(self.shouldRedirect, "/THERE")
3308 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3309 d.addCallback(self.failUnlessNodeKeysAre, [])
3312 def test_POST_bad_t(self):
3313 d = self.shouldFail2(error.Error, "POST_bad_t",
3315 "POST to a directory with bad t=BOGUS",
3316 self.POST, self.public_url + "/foo", t="BOGUS")
3319 def test_POST_set_children(self, command_name="set_children"):
3320 contents9, n9, newuri9 = self.makefile(9)
3321 contents10, n10, newuri10 = self.makefile(10)
3322 contents11, n11, newuri11 = self.makefile(11)
3325 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3328 "ctime": 1002777696.7564139,
3329 "mtime": 1002777696.7564139
3332 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3335 "ctime": 1002777696.7564139,
3336 "mtime": 1002777696.7564139
3339 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3342 "ctime": 1002777696.7564139,
3343 "mtime": 1002777696.7564139
3346 }""" % (newuri9, newuri10, newuri11)
3348 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3350 d = client.getPage(url, method="POST", postdata=reqbody)
3352 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3353 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3354 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3356 d.addCallback(_then)
3357 d.addErrback(self.dump_error)
3360 def test_POST_set_children_with_hyphen(self):
3361 return self.test_POST_set_children(command_name="set-children")
3363 def test_POST_link_uri(self):
3364 contents, n, newuri = self.makefile(8)
3365 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3366 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3367 d.addCallback(lambda res:
3368 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3372 def test_POST_link_uri_replace(self):
3373 contents, n, newuri = self.makefile(8)
3374 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3375 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3376 d.addCallback(lambda res:
3377 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3381 def test_POST_link_uri_unknown_bad(self):
3382 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3383 d.addBoth(self.shouldFail, error.Error,
3384 "POST_link_uri_unknown_bad",
3386 "unknown cap in a write slot")
3389 def test_POST_link_uri_unknown_ro_good(self):
3390 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3391 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3394 def test_POST_link_uri_unknown_imm_good(self):
3395 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3396 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3399 def test_POST_link_uri_no_replace_queryarg(self):
3400 contents, n, newuri = self.makefile(8)
3401 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3402 name="bar.txt", uri=newuri)
3403 d.addBoth(self.shouldFail, error.Error,
3404 "POST_link_uri_no_replace_queryarg",
3406 "There was already a child by that name, and you asked me "
3407 "to not replace it")
3408 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3409 d.addCallback(self.failUnlessIsBarDotTxt)
3412 def test_POST_link_uri_no_replace_field(self):
3413 contents, n, newuri = self.makefile(8)
3414 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3415 name="bar.txt", uri=newuri)
3416 d.addBoth(self.shouldFail, error.Error,
3417 "POST_link_uri_no_replace_field",
3419 "There was already a child by that name, and you asked me "
3420 "to not replace it")
3421 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3422 d.addCallback(self.failUnlessIsBarDotTxt)
3425 def test_POST_delete(self, command_name='delete'):
3426 d = self._foo_node.list()
3427 def _check_before(children):
3428 self.failUnlessIn(u"bar.txt", children)
3429 d.addCallback(_check_before)
3430 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3431 d.addCallback(lambda res: self._foo_node.list())
3432 def _check_after(children):
3433 self.failIfIn(u"bar.txt", children)
3434 d.addCallback(_check_after)
3437 def test_POST_unlink(self):
3438 return self.test_POST_delete(command_name='unlink')
3440 def test_POST_rename_file(self):
3441 d = self.POST(self.public_url + "/foo", t="rename",
3442 from_name="bar.txt", to_name='wibble.txt')
3443 d.addCallback(lambda res:
3444 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3445 d.addCallback(lambda res:
3446 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3447 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3448 d.addCallback(self.failUnlessIsBarDotTxt)
3449 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3450 d.addCallback(self.failUnlessIsBarJSON)
3453 def test_POST_rename_file_redundant(self):
3454 d = self.POST(self.public_url + "/foo", t="rename",
3455 from_name="bar.txt", to_name='bar.txt')
3456 d.addCallback(lambda res:
3457 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3458 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3459 d.addCallback(self.failUnlessIsBarDotTxt)
3460 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3461 d.addCallback(self.failUnlessIsBarJSON)
3464 def test_POST_rename_file_replace(self):
3465 # rename a file and replace a directory with it
3466 d = self.POST(self.public_url + "/foo", t="rename",
3467 from_name="bar.txt", to_name='empty')
3468 d.addCallback(lambda res:
3469 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3470 d.addCallback(lambda res:
3471 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3472 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3473 d.addCallback(self.failUnlessIsBarDotTxt)
3474 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3475 d.addCallback(self.failUnlessIsBarJSON)
3478 def test_POST_rename_file_no_replace_queryarg(self):
3479 # rename a file and replace a directory with it
3480 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3481 from_name="bar.txt", to_name='empty')
3482 d.addBoth(self.shouldFail, error.Error,
3483 "POST_rename_file_no_replace_queryarg",
3485 "There was already a child by that name, and you asked me "
3486 "to not replace it")
3487 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3488 d.addCallback(self.failUnlessIsEmptyJSON)
3491 def test_POST_rename_file_no_replace_field(self):
3492 # rename a file and replace a directory with it
3493 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3494 from_name="bar.txt", to_name='empty')
3495 d.addBoth(self.shouldFail, error.Error,
3496 "POST_rename_file_no_replace_field",
3498 "There was already a child by that name, and you asked me "
3499 "to not replace it")
3500 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3501 d.addCallback(self.failUnlessIsEmptyJSON)
3504 def failUnlessIsEmptyJSON(self, res):
3505 data = simplejson.loads(res)
3506 self.failUnlessEqual(data[0], "dirnode", data)
3507 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3509 def test_POST_rename_file_slash_fail(self):
3510 d = self.POST(self.public_url + "/foo", t="rename",
3511 from_name="bar.txt", to_name='kirk/spock.txt')
3512 d.addBoth(self.shouldFail, error.Error,
3513 "test_POST_rename_file_slash_fail",
3515 "to_name= may not contain a slash",
3517 d.addCallback(lambda res:
3518 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3521 def test_POST_rename_dir(self):
3522 d = self.POST(self.public_url, t="rename",
3523 from_name="foo", to_name='plunk')
3524 d.addCallback(lambda res:
3525 self.failIfNodeHasChild(self.public_root, u"foo"))
3526 d.addCallback(lambda res:
3527 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3528 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3529 d.addCallback(self.failUnlessIsFooJSON)
3532 def test_POST_move_file(self):
3533 d = self.POST(self.public_url + "/foo", t="move",
3534 from_name="bar.txt", to_dir="sub")
3535 d.addCallback(lambda res:
3536 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3537 d.addCallback(lambda res:
3538 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3539 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3540 d.addCallback(self.failUnlessIsBarDotTxt)
3541 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3542 d.addCallback(self.failUnlessIsBarJSON)
3545 def test_POST_move_file_new_name(self):
3546 d = self.POST(self.public_url + "/foo", t="move",
3547 from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3548 d.addCallback(lambda res:
3549 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3550 d.addCallback(lambda res:
3551 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3552 d.addCallback(lambda res:
3553 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3554 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3555 d.addCallback(self.failUnlessIsBarDotTxt)
3556 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3557 d.addCallback(self.failUnlessIsBarJSON)
3560 def test_POST_move_file_replace(self):
3561 d = self.POST(self.public_url + "/foo", t="move",
3562 from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3563 d.addCallback(lambda res:
3564 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3565 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3566 d.addCallback(self.failUnlessIsBarDotTxt)
3567 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3568 d.addCallback(self.failUnlessIsBarJSON)
3571 def test_POST_move_file_no_replace(self):
3572 d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3574 "There was already a child by that name, and you asked me to not replace it",
3575 self.POST, self.public_url + "/foo", t="move",
3576 replace="false", from_name="bar.txt",
3577 to_name="baz.txt", to_dir="sub")
3578 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3579 d.addCallback(self.failUnlessIsBarDotTxt)
3580 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3581 d.addCallback(self.failUnlessIsBarJSON)
3582 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3583 d.addCallback(self.failUnlessIsSubBazDotTxt)
3586 def test_POST_move_file_slash_fail(self):
3587 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3589 "to_name= may not contain a slash",
3590 self.POST, self.public_url + "/foo", t="move",
3591 from_name="bar.txt",
3592 to_name="slash/fail.txt", to_dir="sub")
3593 d.addCallback(lambda res:
3594 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3595 d.addCallback(lambda res:
3596 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3597 d.addCallback(lambda ign:
3598 self.shouldFail2(error.Error,
3599 "test_POST_rename_file_slash_fail2",
3601 "from_name= may not contain a slash",
3602 self.POST, self.public_url + "/foo",
3604 from_name="nope/bar.txt",
3605 to_name="fail.txt", to_dir="sub"))
3608 def test_POST_move_file_no_target(self):
3609 d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3611 "move requires from_name and to_dir",
3612 self.POST, self.public_url + "/foo", t="move",
3613 from_name="bar.txt", to_name="baz.txt")
3614 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3615 d.addCallback(self.failUnlessIsBarDotTxt)
3616 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3617 d.addCallback(self.failUnlessIsBarJSON)
3618 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3619 d.addCallback(self.failUnlessIsBazDotTxt)
3622 def test_POST_move_file_bad_target_type(self):
3623 d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3624 "400 Bad Request", "invalid target_type parameter",
3626 self.public_url + "/foo", t="move",
3627 target_type="*D", from_name="bar.txt",
3631 def test_POST_move_file_multi_level(self):
3632 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3633 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3634 from_name="bar.txt", to_dir="sub/level2"))
3635 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3636 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3637 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3638 d.addCallback(self.failUnlessIsBarDotTxt)
3639 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3640 d.addCallback(self.failUnlessIsBarJSON)
3643 def test_POST_move_file_to_uri(self):
3644 d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3645 from_name="bar.txt", to_dir=self._sub_uri)
3646 d.addCallback(lambda res:
3647 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3648 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3649 d.addCallback(self.failUnlessIsBarDotTxt)
3650 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3651 d.addCallback(self.failUnlessIsBarJSON)
3654 def test_POST_move_file_to_nonexist_dir(self):
3655 d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3656 "404 Not Found", "No such child: nopechucktesta",
3657 self.POST, self.public_url + "/foo", t="move",
3658 from_name="bar.txt", to_dir="nopechucktesta")
3661 def test_POST_move_file_into_file(self):
3662 d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3663 "400 Bad Request", "to_dir is not a directory",
3664 self.POST, self.public_url + "/foo", t="move",
3665 from_name="bar.txt", to_dir="baz.txt")
3666 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3667 d.addCallback(self.failUnlessIsBazDotTxt)
3668 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3669 d.addCallback(self.failUnlessIsBarDotTxt)
3670 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3671 d.addCallback(self.failUnlessIsBarJSON)
3674 def test_POST_move_file_to_bad_uri(self):
3675 d = self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3676 "400 Bad Request", "to_dir is not a directory",
3677 self.POST, self.public_url + "/foo", t="move",
3678 from_name="bar.txt", target_type="uri",
3679 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3680 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3681 d.addCallback(self.failUnlessIsBarDotTxt)
3682 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3683 d.addCallback(self.failUnlessIsBarJSON)
3686 def test_POST_move_dir(self):
3687 d = self.POST(self.public_url + "/foo", t="move",
3688 from_name="bar.txt", to_dir="empty")
3689 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3690 t="move", from_name="empty", to_dir="sub"))
3691 d.addCallback(lambda res:
3692 self.failIfNodeHasChild(self._foo_node, u"empty"))
3693 d.addCallback(lambda res:
3694 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3695 d.addCallback(lambda res:
3696 self._sub_node.get_child_at_path(u"empty"))
3697 d.addCallback(lambda node:
3698 self.failUnlessNodeHasChild(node, u"bar.txt"))
3699 d.addCallback(lambda res:
3700 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3701 d.addCallback(self.failUnlessIsBarDotTxt)
3704 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3705 """ If target is not None then the redirection has to go to target. If
3706 statuscode is not None then the redirection has to be accomplished with
3707 that HTTP status code."""
3708 if not isinstance(res, failure.Failure):
3709 to_where = (target is None) and "somewhere" or ("to " + target)
3710 self.fail("%s: we were expecting to get redirected %s, not get an"
3711 " actual page: %s" % (which, to_where, res))
3712 res.trap(error.PageRedirect)
3713 if statuscode is not None:
3714 self.failUnlessReallyEqual(res.value.status, statuscode,
3715 "%s: not a redirect" % which)
3716 if target is not None:
3717 # the PageRedirect does not seem to capture the uri= query arg
3718 # properly, so we can't check for it.
3719 realtarget = self.webish_url + target
3720 self.failUnlessReallyEqual(res.value.location, realtarget,
3721 "%s: wrong target" % which)
3722 return res.value.location
3724 def test_GET_URI_form(self):
3725 base = "/uri?uri=%s" % self._bar_txt_uri
3726 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3727 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3729 d.addBoth(self.shouldRedirect, targetbase)
3730 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3731 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3732 d.addCallback(lambda res: self.GET(base+"&t=json"))
3733 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3734 d.addCallback(self.log, "about to get file by uri")
3735 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3736 d.addCallback(self.failUnlessIsBarDotTxt)
3737 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3738 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3739 followRedirect=True))
3740 d.addCallback(self.failUnlessIsFooJSON)
3741 d.addCallback(self.log, "got dir by uri")
3745 def test_GET_URI_form_bad(self):
3746 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3747 "400 Bad Request", "GET /uri requires uri=",
3751 def test_GET_rename_form(self):
3752 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3753 followRedirect=True)
3755 self.failUnlessIn('name="when_done" value="."', res)
3756 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3757 self.failUnlessIn(FAVICON_MARKUP, res)
3758 d.addCallback(_check)
3761 def log(self, res, msg):
3762 #print "MSG: %s RES: %s" % (msg, res)
3766 def test_GET_URI_URL(self):
3767 base = "/uri/%s" % self._bar_txt_uri
3769 d.addCallback(self.failUnlessIsBarDotTxt)
3770 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3771 d.addCallback(self.failUnlessIsBarDotTxt)
3772 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3773 d.addCallback(self.failUnlessIsBarDotTxt)
3776 def test_GET_URI_URL_dir(self):
3777 base = "/uri/%s?t=json" % self._foo_uri
3779 d.addCallback(self.failUnlessIsFooJSON)
3782 def test_GET_URI_URL_missing(self):
3783 base = "/uri/%s" % self._bad_file_uri
3784 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3785 http.GONE, None, "NotEnoughSharesError",
3787 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3788 # here? we must arrange for a download to fail after target.open()
3789 # has been called, and then inspect the response to see that it is
3790 # shorter than we expected.
3793 def test_PUT_DIRURL_uri(self):
3794 d = self.s.create_dirnode()
3796 new_uri = dn.get_uri()
3797 # replace /foo with a new (empty) directory
3798 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3799 d.addCallback(lambda res:
3800 self.failUnlessReallyEqual(res.strip(), new_uri))
3801 d.addCallback(lambda res:
3802 self.failUnlessRWChildURIIs(self.public_root,
3806 d.addCallback(_made_dir)
3809 def test_PUT_DIRURL_uri_noreplace(self):
3810 d = self.s.create_dirnode()
3812 new_uri = dn.get_uri()
3813 # replace /foo with a new (empty) directory, but ask that
3814 # replace=false, so it should fail
3815 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3816 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3818 self.public_url + "/foo?t=uri&replace=false",
3820 d.addCallback(lambda res:
3821 self.failUnlessRWChildURIIs(self.public_root,
3825 d.addCallback(_made_dir)
3828 def test_PUT_DIRURL_bad_t(self):
3829 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3830 "400 Bad Request", "PUT to a directory",
3831 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3832 d.addCallback(lambda res:
3833 self.failUnlessRWChildURIIs(self.public_root,
3838 def test_PUT_NEWFILEURL_uri(self):
3839 contents, n, new_uri = self.makefile(8)
3840 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3841 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3842 d.addCallback(lambda res:
3843 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3847 def test_PUT_NEWFILEURL_mdmf(self):
3848 new_contents = self.NEWFILE_CONTENTS * 300000
3849 d = self.PUT(self.public_url + \
3850 "/foo/mdmf.txt?format=mdmf",
3852 d.addCallback(lambda ignored:
3853 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3854 def _got_json(json):
3855 data = simplejson.loads(json)
3857 self.failUnlessIn("format", data)
3858 self.failUnlessEqual(data["format"], "MDMF")
3859 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3860 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3861 d.addCallback(_got_json)
3864 def test_PUT_NEWFILEURL_sdmf(self):
3865 new_contents = self.NEWFILE_CONTENTS * 300000
3866 d = self.PUT(self.public_url + \
3867 "/foo/sdmf.txt?format=sdmf",
3869 d.addCallback(lambda ignored:
3870 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3871 def _got_json(json):
3872 data = simplejson.loads(json)
3874 self.failUnlessIn("format", data)
3875 self.failUnlessEqual(data["format"], "SDMF")
3876 d.addCallback(_got_json)
3879 def test_PUT_NEWFILEURL_bad_format(self):
3880 new_contents = self.NEWFILE_CONTENTS * 300000
3881 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3882 400, "Bad Request", "Unknown format: foo",
3883 self.PUT, self.public_url + \
3884 "/foo/foo.txt?format=foo",
3887 def test_PUT_NEWFILEURL_uri_replace(self):
3888 contents, n, new_uri = self.makefile(8)
3889 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3890 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3891 d.addCallback(lambda res:
3892 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3896 def test_PUT_NEWFILEURL_uri_no_replace(self):
3897 contents, n, new_uri = self.makefile(8)
3898 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3899 d.addBoth(self.shouldFail, error.Error,
3900 "PUT_NEWFILEURL_uri_no_replace",
3902 "There was already a child by that name, and you asked me "
3903 "to not replace it")
3906 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3907 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3908 d.addBoth(self.shouldFail, error.Error,
3909 "POST_put_uri_unknown_bad",
3911 "unknown cap in a write slot")
3914 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3915 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3916 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3917 u"put-future-ro.txt")
3920 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3921 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3922 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3923 u"put-future-imm.txt")
3926 def test_PUT_NEWFILE_URI(self):
3927 file_contents = "New file contents here\n"
3928 d = self.PUT("/uri", file_contents)
3930 assert isinstance(uri, str), uri
3931 self.failUnlessIn(uri, self.get_all_contents())
3932 self.failUnlessReallyEqual(self.get_all_contents()[uri],
3934 return self.GET("/uri/%s" % uri)
3935 d.addCallback(_check)
3937 self.failUnlessReallyEqual(res, file_contents)
3938 d.addCallback(_check2)
3941 def test_PUT_NEWFILE_URI_not_mutable(self):
3942 file_contents = "New file contents here\n"
3943 d = self.PUT("/uri?mutable=false", file_contents)
3945 assert isinstance(uri, str), uri
3946 self.failUnlessIn(uri, self.get_all_contents())
3947 self.failUnlessReallyEqual(self.get_all_contents()[uri],
3949 return self.GET("/uri/%s" % uri)
3950 d.addCallback(_check)
3952 self.failUnlessReallyEqual(res, file_contents)
3953 d.addCallback(_check2)
3956 def test_PUT_NEWFILE_URI_only_PUT(self):
3957 d = self.PUT("/uri?t=bogus", "")
3958 d.addBoth(self.shouldFail, error.Error,
3959 "PUT_NEWFILE_URI_only_PUT",
3961 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3964 def test_PUT_NEWFILE_URI_mutable(self):
3965 file_contents = "New file contents here\n"
3966 d = self.PUT("/uri?mutable=true", file_contents)
3967 def _check1(filecap):
3968 filecap = filecap.strip()
3969 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3970 self.filecap = filecap
3971 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3972 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3973 n = self.s.create_node_from_uri(filecap)
3974 return n.download_best_version()
3975 d.addCallback(_check1)
3977 self.failUnlessReallyEqual(data, file_contents)
3978 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3979 d.addCallback(_check2)
3981 self.failUnlessReallyEqual(res, file_contents)
3982 d.addCallback(_check3)
3985 def test_PUT_mkdir(self):
3986 d = self.PUT("/uri?t=mkdir", "")
3988 n = self.s.create_node_from_uri(uri.strip())
3989 d2 = self.failUnlessNodeKeysAre(n, [])
3990 d2.addCallback(lambda res:
3991 self.GET("/uri/%s?t=json" % uri))
3993 d.addCallback(_check)
3994 d.addCallback(self.failUnlessIsEmptyJSON)
3997 def test_PUT_mkdir_mdmf(self):
3998 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4000 u = uri.from_string(res)
4001 # Check that this is an MDMF writecap
4002 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4006 def test_PUT_mkdir_sdmf(self):
4007 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4009 u = uri.from_string(res)
4010 self.failUnlessIsInstance(u, uri.DirectoryURI)
4014 def test_PUT_mkdir_bad_format(self):
4015 return self.shouldHTTPError("PUT_mkdir_bad_format",
4016 400, "Bad Request", "Unknown format: foo",
4017 self.PUT, "/uri?t=mkdir&format=foo",
4020 def test_POST_check(self):
4021 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4023 # this returns a string form of the results, which are probably
4024 # None since we're using fake filenodes.
4025 # TODO: verify that the check actually happened, by changing
4026 # FakeCHKFileNode to count how many times .check() has been
4029 d.addCallback(_done)
4033 def test_PUT_update_at_offset(self):
4034 file_contents = "test file" * 100000 # about 900 KiB
4035 d = self.PUT("/uri?mutable=true", file_contents)
4037 self.filecap = filecap
4038 new_data = file_contents[:100]
4039 new = "replaced and so on"
4041 new_data += file_contents[len(new_data):]
4042 assert len(new_data) == len(file_contents)
4043 self.new_data = new_data
4044 d.addCallback(_then)
4045 d.addCallback(lambda ignored:
4046 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4047 "replaced and so on"))
4048 def _get_data(filecap):
4049 n = self.s.create_node_from_uri(filecap)
4050 return n.download_best_version()
4051 d.addCallback(_get_data)
4052 d.addCallback(lambda results:
4053 self.failUnlessEqual(results, self.new_data))
4054 # Now try appending things to the file
4055 d.addCallback(lambda ignored:
4056 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4058 d.addCallback(_get_data)
4059 d.addCallback(lambda results:
4060 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4061 # and try replacing the beginning of the file
4062 d.addCallback(lambda ignored:
4063 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4064 d.addCallback(_get_data)
4065 d.addCallback(lambda results:
4066 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4069 def test_PUT_update_at_invalid_offset(self):
4070 file_contents = "test file" * 100000 # about 900 KiB
4071 d = self.PUT("/uri?mutable=true", file_contents)
4073 self.filecap = filecap
4074 d.addCallback(_then)
4075 # Negative offsets should cause an error.
4076 d.addCallback(lambda ignored:
4077 self.shouldHTTPError("PUT_update_at_invalid_offset",
4081 "/uri/%s?offset=-1" % self.filecap,
4085 def test_PUT_update_at_offset_immutable(self):
4086 file_contents = "Test file" * 100000
4087 d = self.PUT("/uri", file_contents)
4089 self.filecap = filecap
4090 d.addCallback(_then)
4091 d.addCallback(lambda ignored:
4092 self.shouldHTTPError("PUT_update_at_offset_immutable",
4096 "/uri/%s?offset=50" % self.filecap,
4101 def test_bad_method(self):
4102 url = self.webish_url + self.public_url + "/foo/bar.txt"
4103 d = self.shouldHTTPError("bad_method",
4104 501, "Not Implemented",
4105 "I don't know how to treat a BOGUS request.",
4106 client.getPage, url, method="BOGUS")
4109 def test_short_url(self):
4110 url = self.webish_url + "/uri"
4111 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4112 "I don't know how to treat a DELETE request.",
4113 client.getPage, url, method="DELETE")
4116 def test_ophandle_bad(self):
4117 url = self.webish_url + "/operations/bogus?t=status"
4118 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4119 "unknown/expired handle 'bogus'",
4120 client.getPage, url)
4123 def test_ophandle_cancel(self):
4124 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4125 followRedirect=True)
4126 d.addCallback(lambda ignored:
4127 self.GET("/operations/128?t=status&output=JSON"))
4129 data = simplejson.loads(res)
4130 self.failUnless("finished" in data, res)
4131 monitor = self.ws.root.child_operations.handles["128"][0]
4132 d = self.POST("/operations/128?t=cancel&output=JSON")
4134 data = simplejson.loads(res)
4135 self.failUnless("finished" in data, res)
4136 # t=cancel causes the handle to be forgotten
4137 self.failUnless(monitor.is_cancelled())
4138 d.addCallback(_check2)
4140 d.addCallback(_check1)
4141 d.addCallback(lambda ignored:
4142 self.shouldHTTPError("ophandle_cancel",
4143 404, "404 Not Found",
4144 "unknown/expired handle '128'",
4146 "/operations/128?t=status&output=JSON"))
4149 def test_ophandle_retainfor(self):
4150 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4151 followRedirect=True)
4152 d.addCallback(lambda ignored:
4153 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4155 data = simplejson.loads(res)
4156 self.failUnless("finished" in data, res)
4157 d.addCallback(_check1)
4158 # the retain-for=0 will cause the handle to be expired very soon
4159 d.addCallback(lambda ign:
4160 self.clock.advance(2.0))
4161 d.addCallback(lambda ignored:
4162 self.shouldHTTPError("ophandle_retainfor",
4163 404, "404 Not Found",
4164 "unknown/expired handle '129'",
4166 "/operations/129?t=status&output=JSON"))
4169 def test_ophandle_release_after_complete(self):
4170 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4171 followRedirect=True)
4172 d.addCallback(self.wait_for_operation, "130")
4173 d.addCallback(lambda ignored:
4174 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4175 # the release-after-complete=true will cause the handle to be expired
4176 d.addCallback(lambda ignored:
4177 self.shouldHTTPError("ophandle_release_after_complete",
4178 404, "404 Not Found",
4179 "unknown/expired handle '130'",
4181 "/operations/130?t=status&output=JSON"))
4184 def test_uncollected_ophandle_expiration(self):
4185 # uncollected ophandles should expire after 4 days
4186 def _make_uncollected_ophandle(ophandle):
4187 d = self.POST(self.public_url +
4188 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4189 followRedirect=False)
4190 # When we start the operation, the webapi server will want
4191 # to redirect us to the page for the ophandle, so we get
4192 # confirmation that the operation has started. If the
4193 # manifest operation has finished by the time we get there,
4194 # following that redirect (by setting followRedirect=True
4195 # above) has the side effect of collecting the ophandle that
4196 # we've just created, which means that we can't use the
4197 # ophandle to test the uncollected timeout anymore. So,
4198 # instead, catch the 302 here and don't follow it.
4199 d.addBoth(self.should302, "uncollected_ophandle_creation")
4201 # Create an ophandle, don't collect it, then advance the clock by
4202 # 4 days - 1 second and make sure that the ophandle is still there.
4203 d = _make_uncollected_ophandle(131)
4204 d.addCallback(lambda ign:
4205 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4206 d.addCallback(lambda ign:
4207 self.GET("/operations/131?t=status&output=JSON"))
4209 data = simplejson.loads(res)
4210 self.failUnless("finished" in data, res)
4211 d.addCallback(_check1)
4212 # Create an ophandle, don't collect it, then try to collect it
4213 # after 4 days. It should be gone.
4214 d.addCallback(lambda ign:
4215 _make_uncollected_ophandle(132))
4216 d.addCallback(lambda ign:
4217 self.clock.advance(96*60*60))
4218 d.addCallback(lambda ign:
4219 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4220 404, "404 Not Found",
4221 "unknown/expired handle '132'",
4223 "/operations/132?t=status&output=JSON"))
4226 def test_collected_ophandle_expiration(self):
4227 # collected ophandles should expire after 1 day
4228 def _make_collected_ophandle(ophandle):
4229 d = self.POST(self.public_url +
4230 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4231 followRedirect=True)
4232 # By following the initial redirect, we collect the ophandle
4233 # we've just created.
4235 # Create a collected ophandle, then collect it after 23 hours
4236 # and 59 seconds to make sure that it is still there.
4237 d = _make_collected_ophandle(133)
4238 d.addCallback(lambda ign:
4239 self.clock.advance((24*60*60) - 1))
4240 d.addCallback(lambda ign:
4241 self.GET("/operations/133?t=status&output=JSON"))
4243 data = simplejson.loads(res)
4244 self.failUnless("finished" in data, res)
4245 d.addCallback(_check1)
4246 # Create another uncollected ophandle, then try to collect it
4247 # after 24 hours to make sure that it is gone.
4248 d.addCallback(lambda ign:
4249 _make_collected_ophandle(134))
4250 d.addCallback(lambda ign:
4251 self.clock.advance(24*60*60))
4252 d.addCallback(lambda ign:
4253 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4254 404, "404 Not Found",
4255 "unknown/expired handle '134'",
4257 "/operations/134?t=status&output=JSON"))
4260 def test_incident(self):
4261 d = self.POST("/report_incident", details="eek")
4263 self.failIfIn("<html>", res)
4264 self.failUnlessIn("Thank you for your report!", res)
4265 d.addCallback(_done)
4268 def test_static(self):
4269 webdir = os.path.join(self.staticdir, "subdir")
4270 fileutil.make_dirs(webdir)
4271 f = open(os.path.join(webdir, "hello.txt"), "wb")
4275 d = self.GET("/static/subdir/hello.txt")
4277 self.failUnlessReallyEqual(res, "hello")
4278 d.addCallback(_check)
4282 class IntroducerWeb(unittest.TestCase):
4287 d = defer.succeed(None)
4289 d.addCallback(lambda ign: self.node.stopService())
4290 d.addCallback(flushEventualQueue)
4293 def test_welcome(self):
4294 basedir = "web.IntroducerWeb.test_welcome"
4296 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4297 self.node = IntroducerNode(basedir)
4298 self.ws = self.node.getServiceNamed("webish")
4300 d = fireEventually(None)
4301 d.addCallback(lambda ign: self.node.startService())
4302 d.addCallback(lambda ign: self.node.when_tub_ready())
4304 d.addCallback(lambda ign: self.GET("/"))
4306 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4307 self.failUnlessIn(FAVICON_MARKUP, res)
4308 d.addCallback(_check)
4311 def GET(self, urlpath, followRedirect=False, return_response=False,
4313 # if return_response=True, this fires with (data, statuscode,
4314 # respheaders) instead of just data.
4315 assert not isinstance(urlpath, unicode)
4316 url = self.ws.getURL().rstrip('/') + urlpath
4317 factory = HTTPClientGETFactory(url, method="GET",
4318 followRedirect=followRedirect, **kwargs)
4319 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4320 d = factory.deferred
4321 def _got_data(data):
4322 return (data, factory.status, factory.response_headers)
4324 d.addCallback(_got_data)
4325 return factory.deferred
4328 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4329 def test_load_file(self):
4330 # This will raise an exception unless a well-formed XML file is found under that name.
4331 common.getxmlfile('directory.xhtml').load()
4333 def test_parse_replace_arg(self):
4334 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4335 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4336 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4338 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4339 common.parse_replace_arg, "only_fles")
4341 def test_abbreviate_time(self):
4342 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4343 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4344 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4345 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4346 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4347 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4349 def test_compute_rate(self):
4350 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4351 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4352 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4353 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4354 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4355 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4356 self.shouldFail(AssertionError, "test_compute_rate", "",
4357 common.compute_rate, -100, 10)
4358 self.shouldFail(AssertionError, "test_compute_rate", "",
4359 common.compute_rate, 100, -10)
4362 rate = common.compute_rate(10*1000*1000, 1)
4363 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4365 def test_abbreviate_rate(self):
4366 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4367 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4368 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4369 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4371 def test_abbreviate_size(self):
4372 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4373 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4374 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4375 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4376 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4378 def test_plural(self):
4380 return "%d second%s" % (s, status.plural(s))
4381 self.failUnlessReallyEqual(convert(0), "0 seconds")
4382 self.failUnlessReallyEqual(convert(1), "1 second")
4383 self.failUnlessReallyEqual(convert(2), "2 seconds")
4385 return "has share%s: %s" % (status.plural(s), ",".join(s))
4386 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4387 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4388 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4391 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4393 def CHECK(self, ign, which, args, clientnum=0):
4394 fileurl = self.fileurls[which]
4395 url = fileurl + "?" + args
4396 return self.GET(url, method="POST", clientnum=clientnum)
4398 def test_filecheck(self):
4399 self.basedir = "web/Grid/filecheck"
4401 c0 = self.g.clients[0]
4404 d = c0.upload(upload.Data(DATA, convergence=""))
4405 def _stash_uri(ur, which):
4406 self.uris[which] = ur.get_uri()
4407 d.addCallback(_stash_uri, "good")
4408 d.addCallback(lambda ign:
4409 c0.upload(upload.Data(DATA+"1", convergence="")))
4410 d.addCallback(_stash_uri, "sick")
4411 d.addCallback(lambda ign:
4412 c0.upload(upload.Data(DATA+"2", convergence="")))
4413 d.addCallback(_stash_uri, "dead")
4414 def _stash_mutable_uri(n, which):
4415 self.uris[which] = n.get_uri()
4416 assert isinstance(self.uris[which], str)
4417 d.addCallback(lambda ign:
4418 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4419 d.addCallback(_stash_mutable_uri, "corrupt")
4420 d.addCallback(lambda ign:
4421 c0.upload(upload.Data("literal", convergence="")))
4422 d.addCallback(_stash_uri, "small")
4423 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4424 d.addCallback(_stash_mutable_uri, "smalldir")
4426 def _compute_fileurls(ignored):
4428 for which in self.uris:
4429 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4430 d.addCallback(_compute_fileurls)
4432 def _clobber_shares(ignored):
4433 good_shares = self.find_uri_shares(self.uris["good"])
4434 self.failUnlessReallyEqual(len(good_shares), 10)
4435 sick_shares = self.find_uri_shares(self.uris["sick"])
4436 os.unlink(sick_shares[0][2])
4437 dead_shares = self.find_uri_shares(self.uris["dead"])
4438 for i in range(1, 10):
4439 os.unlink(dead_shares[i][2])
4440 c_shares = self.find_uri_shares(self.uris["corrupt"])
4441 cso = CorruptShareOptions()
4442 cso.stdout = StringIO()
4443 cso.parseOptions([c_shares[0][2]])
4445 d.addCallback(_clobber_shares)
4447 d.addCallback(self.CHECK, "good", "t=check")
4448 def _got_html_good(res):
4449 self.failUnlessIn("Healthy", res)
4450 self.failIfIn("Not Healthy", res)
4451 self.failUnlessIn(FAVICON_MARKUP, res)
4452 d.addCallback(_got_html_good)
4453 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4454 def _got_html_good_return_to(res):
4455 self.failUnlessIn("Healthy", res)
4456 self.failIfIn("Not Healthy", res)
4457 self.failUnlessIn('<a href="somewhere">Return to file', res)
4458 d.addCallback(_got_html_good_return_to)
4459 d.addCallback(self.CHECK, "good", "t=check&output=json")
4460 def _got_json_good(res):
4461 r = simplejson.loads(res)
4462 self.failUnlessEqual(r["summary"], "Healthy")
4463 self.failUnless(r["results"]["healthy"])
4464 self.failIf(r["results"]["needs-rebalancing"])
4465 self.failUnless(r["results"]["recoverable"])
4466 d.addCallback(_got_json_good)
4468 d.addCallback(self.CHECK, "small", "t=check")
4469 def _got_html_small(res):
4470 self.failUnlessIn("Literal files are always healthy", res)
4471 self.failIfIn("Not Healthy", res)
4472 d.addCallback(_got_html_small)
4473 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4474 def _got_html_small_return_to(res):
4475 self.failUnlessIn("Literal files are always healthy", res)
4476 self.failIfIn("Not Healthy", res)
4477 self.failUnlessIn('<a href="somewhere">Return to file', res)
4478 d.addCallback(_got_html_small_return_to)
4479 d.addCallback(self.CHECK, "small", "t=check&output=json")
4480 def _got_json_small(res):
4481 r = simplejson.loads(res)
4482 self.failUnlessEqual(r["storage-index"], "")
4483 self.failUnless(r["results"]["healthy"])
4484 d.addCallback(_got_json_small)
4486 d.addCallback(self.CHECK, "smalldir", "t=check")
4487 def _got_html_smalldir(res):
4488 self.failUnlessIn("Literal files are always healthy", res)
4489 self.failIfIn("Not Healthy", res)
4490 d.addCallback(_got_html_smalldir)
4491 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4492 def _got_json_smalldir(res):
4493 r = simplejson.loads(res)
4494 self.failUnlessEqual(r["storage-index"], "")
4495 self.failUnless(r["results"]["healthy"])
4496 d.addCallback(_got_json_smalldir)
4498 d.addCallback(self.CHECK, "sick", "t=check")
4499 def _got_html_sick(res):
4500 self.failUnlessIn("Not Healthy", res)
4501 d.addCallback(_got_html_sick)
4502 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4503 def _got_json_sick(res):
4504 r = simplejson.loads(res)
4505 self.failUnlessEqual(r["summary"],
4506 "Not Healthy: 9 shares (enc 3-of-10)")
4507 self.failIf(r["results"]["healthy"])
4508 self.failIf(r["results"]["needs-rebalancing"])
4509 self.failUnless(r["results"]["recoverable"])
4510 d.addCallback(_got_json_sick)
4512 d.addCallback(self.CHECK, "dead", "t=check")
4513 def _got_html_dead(res):
4514 self.failUnlessIn("Not Healthy", res)
4515 d.addCallback(_got_html_dead)
4516 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4517 def _got_json_dead(res):
4518 r = simplejson.loads(res)
4519 self.failUnlessEqual(r["summary"],
4520 "Not Healthy: 1 shares (enc 3-of-10)")
4521 self.failIf(r["results"]["healthy"])
4522 self.failIf(r["results"]["needs-rebalancing"])
4523 self.failIf(r["results"]["recoverable"])
4524 d.addCallback(_got_json_dead)
4526 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4527 def _got_html_corrupt(res):
4528 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4529 d.addCallback(_got_html_corrupt)
4530 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4531 def _got_json_corrupt(res):
4532 r = simplejson.loads(res)
4533 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4534 self.failIf(r["results"]["healthy"])
4535 self.failUnless(r["results"]["recoverable"])
4536 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4537 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4538 d.addCallback(_got_json_corrupt)
4540 d.addErrback(self.explain_web_error)
4543 def test_repair_html(self):
4544 self.basedir = "web/Grid/repair_html"
4546 c0 = self.g.clients[0]
4549 d = c0.upload(upload.Data(DATA, convergence=""))
4550 def _stash_uri(ur, which):
4551 self.uris[which] = ur.get_uri()
4552 d.addCallback(_stash_uri, "good")
4553 d.addCallback(lambda ign:
4554 c0.upload(upload.Data(DATA+"1", convergence="")))
4555 d.addCallback(_stash_uri, "sick")
4556 d.addCallback(lambda ign:
4557 c0.upload(upload.Data(DATA+"2", convergence="")))
4558 d.addCallback(_stash_uri, "dead")
4559 def _stash_mutable_uri(n, which):
4560 self.uris[which] = n.get_uri()
4561 assert isinstance(self.uris[which], str)
4562 d.addCallback(lambda ign:
4563 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4564 d.addCallback(_stash_mutable_uri, "corrupt")
4566 def _compute_fileurls(ignored):
4568 for which in self.uris:
4569 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4570 d.addCallback(_compute_fileurls)
4572 def _clobber_shares(ignored):
4573 good_shares = self.find_uri_shares(self.uris["good"])
4574 self.failUnlessReallyEqual(len(good_shares), 10)
4575 sick_shares = self.find_uri_shares(self.uris["sick"])
4576 os.unlink(sick_shares[0][2])
4577 dead_shares = self.find_uri_shares(self.uris["dead"])
4578 for i in range(1, 10):
4579 os.unlink(dead_shares[i][2])
4580 c_shares = self.find_uri_shares(self.uris["corrupt"])
4581 cso = CorruptShareOptions()
4582 cso.stdout = StringIO()
4583 cso.parseOptions([c_shares[0][2]])
4585 d.addCallback(_clobber_shares)
4587 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4588 def _got_html_good(res):
4589 self.failUnlessIn("Healthy", res)
4590 self.failIfIn("Not Healthy", res)
4591 self.failUnlessIn("No repair necessary", res)
4592 self.failUnlessIn(FAVICON_MARKUP, res)
4593 d.addCallback(_got_html_good)
4595 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4596 def _got_html_sick(res):
4597 self.failUnlessIn("Healthy : healthy", res)
4598 self.failIfIn("Not Healthy", res)
4599 self.failUnlessIn("Repair successful", res)
4600 d.addCallback(_got_html_sick)
4602 # repair of a dead file will fail, of course, but it isn't yet
4603 # clear how this should be reported. Right now it shows up as
4606 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4607 #def _got_html_dead(res):
4609 # self.failUnlessIn("Healthy : healthy", res)
4610 # self.failIfIn("Not Healthy", res)
4611 # self.failUnlessIn("No repair necessary", res)
4612 #d.addCallback(_got_html_dead)
4614 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4615 def _got_html_corrupt(res):
4616 self.failUnlessIn("Healthy : Healthy", res)
4617 self.failIfIn("Not Healthy", res)
4618 self.failUnlessIn("Repair successful", res)
4619 d.addCallback(_got_html_corrupt)
4621 d.addErrback(self.explain_web_error)
4624 def test_repair_json(self):
4625 self.basedir = "web/Grid/repair_json"
4627 c0 = self.g.clients[0]
4630 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4631 def _stash_uri(ur, which):
4632 self.uris[which] = ur.get_uri()
4633 d.addCallback(_stash_uri, "sick")
4635 def _compute_fileurls(ignored):
4637 for which in self.uris:
4638 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4639 d.addCallback(_compute_fileurls)
4641 def _clobber_shares(ignored):
4642 sick_shares = self.find_uri_shares(self.uris["sick"])
4643 os.unlink(sick_shares[0][2])
4644 d.addCallback(_clobber_shares)
4646 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4647 def _got_json_sick(res):
4648 r = simplejson.loads(res)
4649 self.failUnlessReallyEqual(r["repair-attempted"], True)
4650 self.failUnlessReallyEqual(r["repair-successful"], True)
4651 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4652 "Not Healthy: 9 shares (enc 3-of-10)")
4653 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4654 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4655 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4656 d.addCallback(_got_json_sick)
4658 d.addErrback(self.explain_web_error)
4661 def test_unknown(self, immutable=False):
4662 self.basedir = "web/Grid/unknown"
4664 self.basedir = "web/Grid/unknown-immutable"
4667 c0 = self.g.clients[0]
4671 # the future cap format may contain slashes, which must be tolerated
4672 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4676 name = u"future-imm"
4677 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4678 d = c0.create_immutable_dirnode({name: (future_node, {})})
4681 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4682 d = c0.create_dirnode()
4684 def _stash_root_and_create_file(n):
4686 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4687 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4689 return self.rootnode.set_node(name, future_node)
4690 d.addCallback(_stash_root_and_create_file)
4692 # make sure directory listing tolerates unknown nodes
4693 d.addCallback(lambda ign: self.GET(self.rooturl))
4694 def _check_directory_html(res, expected_type_suffix):
4695 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4696 '<td>%s</td>' % (expected_type_suffix, str(name)),
4698 self.failUnless(re.search(pattern, res), res)
4699 # find the More Info link for name, should be relative
4700 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4701 info_url = mo.group(1)
4702 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4704 d.addCallback(_check_directory_html, "-IMM")
4706 d.addCallback(_check_directory_html, "")
4708 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4709 def _check_directory_json(res, expect_rw_uri):
4710 data = simplejson.loads(res)
4711 self.failUnlessEqual(data[0], "dirnode")
4712 f = data[1]["children"][name]
4713 self.failUnlessEqual(f[0], "unknown")
4715 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4717 self.failIfIn("rw_uri", f[1])
4719 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4721 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4722 self.failUnlessIn("metadata", f[1])
4723 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4725 def _check_info(res, expect_rw_uri, expect_ro_uri):
4726 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4728 self.failUnlessIn(unknown_rwcap, res)
4731 self.failUnlessIn(unknown_immcap, res)
4733 self.failUnlessIn(unknown_rocap, res)
4735 self.failIfIn(unknown_rocap, res)
4736 self.failIfIn("Raw data as", res)
4737 self.failIfIn("Directory writecap", res)
4738 self.failIfIn("Checker Operations", res)
4739 self.failIfIn("Mutable File Operations", res)
4740 self.failIfIn("Directory Operations", res)
4742 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4743 # why they fail. Possibly related to ticket #922.
4745 d.addCallback(lambda ign: self.GET(expected_info_url))
4746 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4747 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4748 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4750 def _check_json(res, expect_rw_uri):
4751 data = simplejson.loads(res)
4752 self.failUnlessEqual(data[0], "unknown")
4754 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4756 self.failIfIn("rw_uri", data[1])
4759 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4760 self.failUnlessReallyEqual(data[1]["mutable"], False)
4762 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4763 self.failUnlessReallyEqual(data[1]["mutable"], True)
4765 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4766 self.failIfIn("mutable", data[1])
4768 # TODO: check metadata contents
4769 self.failUnlessIn("metadata", data[1])
4771 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4772 d.addCallback(_check_json, expect_rw_uri=not immutable)
4774 # and make sure that a read-only version of the directory can be
4775 # rendered too. This version will not have unknown_rwcap, whether
4776 # or not future_node was immutable.
4777 d.addCallback(lambda ign: self.GET(self.rourl))
4779 d.addCallback(_check_directory_html, "-IMM")
4781 d.addCallback(_check_directory_html, "-RO")
4783 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4784 d.addCallback(_check_directory_json, expect_rw_uri=False)
4786 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4787 d.addCallback(_check_json, expect_rw_uri=False)
4789 # TODO: check that getting t=info from the Info link in the ro directory
4790 # works, and does not include the writecap URI.
4793 def test_immutable_unknown(self):
4794 return self.test_unknown(immutable=True)
4796 def test_mutant_dirnodes_are_omitted(self):
4797 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4800 c = self.g.clients[0]
4805 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4806 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4807 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4809 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4810 # test the dirnode and web layers separately.
4812 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4813 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4814 # When the directory is read, the mutants should be silently disposed of, leaving
4815 # their lonely sibling.
4816 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4817 # because immutable directories don't have a writecap and therefore that field
4818 # isn't (and can't be) decrypted.
4819 # TODO: The field still exists in the netstring. Technically we should check what
4820 # happens if something is put there (_unpack_contents should raise ValueError),
4821 # but that can wait.
4823 lonely_child = nm.create_from_cap(lonely_uri)
4824 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4825 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4827 def _by_hook_or_by_crook():
4829 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4830 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4832 mutant_write_in_ro_child.get_write_uri = lambda: None
4833 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4835 kids = {u"lonely": (lonely_child, {}),
4836 u"ro": (mutant_ro_child, {}),
4837 u"write-in-ro": (mutant_write_in_ro_child, {}),
4839 d = c.create_immutable_dirnode(kids)
4842 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4843 self.failIf(dn.is_mutable())
4844 self.failUnless(dn.is_readonly())
4845 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4846 self.failIf(hasattr(dn._node, 'get_writekey'))
4848 self.failUnlessIn("RO-IMM", rep)
4850 self.failUnlessIn("CHK", cap.to_string())
4853 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4854 return download_to_data(dn._node)
4855 d.addCallback(_created)
4857 def _check_data(data):
4858 # Decode the netstring representation of the directory to check that all children
4859 # are present. This is a bit of an abstraction violation, but there's not really
4860 # any other way to do it given that the real DirectoryNode._unpack_contents would
4861 # strip the mutant children out (which is what we're trying to test, later).
4864 while position < len(data):
4865 entries, position = split_netstring(data, 1, position)
4867 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4868 name = name_utf8.decode("utf-8")
4869 self.failUnlessEqual(rwcapdata, "")
4870 self.failUnlessIn(name, kids)
4871 (expected_child, ign) = kids[name]
4872 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4875 self.failUnlessReallyEqual(numkids, 3)
4876 return self.rootnode.list()
4877 d.addCallback(_check_data)
4879 # Now when we use the real directory listing code, the mutants should be absent.
4880 def _check_kids(children):
4881 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4882 lonely_node, lonely_metadata = children[u"lonely"]
4884 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4885 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4886 d.addCallback(_check_kids)
4888 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4889 d.addCallback(lambda n: n.list())
4890 d.addCallback(_check_kids) # again with dirnode recreated from cap
4892 # Make sure the lonely child can be listed in HTML...
4893 d.addCallback(lambda ign: self.GET(self.rooturl))
4894 def _check_html(res):
4895 self.failIfIn("URI:SSK", res)
4896 get_lonely = "".join([r'<td>FILE</td>',
4898 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4900 r'\s+<td align="right">%d</td>' % len("one"),
4902 self.failUnless(re.search(get_lonely, res), res)
4904 # find the More Info link for name, should be relative
4905 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4906 info_url = mo.group(1)
4907 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4908 d.addCallback(_check_html)
4911 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4912 def _check_json(res):
4913 data = simplejson.loads(res)
4914 self.failUnlessEqual(data[0], "dirnode")
4915 listed_children = data[1]["children"]
4916 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4917 ll_type, ll_data = listed_children[u"lonely"]
4918 self.failUnlessEqual(ll_type, "filenode")
4919 self.failIfIn("rw_uri", ll_data)
4920 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4921 d.addCallback(_check_json)
4924 def test_deep_check(self):
4925 self.basedir = "web/Grid/deep_check"
4927 c0 = self.g.clients[0]
4931 d = c0.create_dirnode()
4932 def _stash_root_and_create_file(n):
4934 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4935 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4936 d.addCallback(_stash_root_and_create_file)
4937 def _stash_uri(fn, which):
4938 self.uris[which] = fn.get_uri()
4940 d.addCallback(_stash_uri, "good")
4941 d.addCallback(lambda ign:
4942 self.rootnode.add_file(u"small",
4943 upload.Data("literal",
4945 d.addCallback(_stash_uri, "small")
4946 d.addCallback(lambda ign:
4947 self.rootnode.add_file(u"sick",
4948 upload.Data(DATA+"1",
4950 d.addCallback(_stash_uri, "sick")
4952 # this tests that deep-check and stream-manifest will ignore
4953 # UnknownNode instances. Hopefully this will also cover deep-stats.
4954 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4955 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4957 def _clobber_shares(ignored):
4958 self.delete_shares_numbered(self.uris["sick"], [0,1])
4959 d.addCallback(_clobber_shares)
4967 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4970 units = [simplejson.loads(line)
4971 for line in res.splitlines()
4974 print "response is:", res
4975 print "undecodeable line was '%s'" % line
4977 self.failUnlessReallyEqual(len(units), 5+1)
4978 # should be parent-first
4980 self.failUnlessEqual(u0["path"], [])
4981 self.failUnlessEqual(u0["type"], "directory")
4982 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4983 u0cr = u0["check-results"]
4984 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4986 ugood = [u for u in units
4987 if u["type"] == "file" and u["path"] == [u"good"]][0]
4988 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4989 ugoodcr = ugood["check-results"]
4990 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4993 self.failUnlessEqual(stats["type"], "stats")
4995 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4996 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4997 self.failUnlessReallyEqual(s["count-directories"], 1)
4998 self.failUnlessReallyEqual(s["count-unknown"], 1)
4999 d.addCallback(_done)
5001 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5002 def _check_manifest(res):
5003 self.failUnless(res.endswith("\n"))
5004 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5005 self.failUnlessReallyEqual(len(units), 5+1)
5006 self.failUnlessEqual(units[-1]["type"], "stats")
5008 self.failUnlessEqual(first["path"], [])
5009 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5010 self.failUnlessEqual(first["type"], "directory")
5011 stats = units[-1]["stats"]
5012 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5013 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5014 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5015 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5016 self.failUnlessReallyEqual(stats["count-unknown"], 1)
5017 d.addCallback(_check_manifest)
5019 # now add root/subdir and root/subdir/grandchild, then make subdir
5020 # unrecoverable, then see what happens
5022 d.addCallback(lambda ign:
5023 self.rootnode.create_subdirectory(u"subdir"))
5024 d.addCallback(_stash_uri, "subdir")
5025 d.addCallback(lambda subdir_node:
5026 subdir_node.add_file(u"grandchild",
5027 upload.Data(DATA+"2",
5029 d.addCallback(_stash_uri, "grandchild")
5031 d.addCallback(lambda ign:
5032 self.delete_shares_numbered(self.uris["subdir"],
5040 # root/subdir [unrecoverable]
5041 # root/subdir/grandchild
5043 # how should a streaming-JSON API indicate fatal error?
5044 # answer: emit ERROR: instead of a JSON string
5046 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5047 def _check_broken_manifest(res):
5048 lines = res.splitlines()
5050 for (i,line) in enumerate(lines)
5051 if line.startswith("ERROR:")]
5053 self.fail("no ERROR: in output: %s" % (res,))
5054 first_error = error_lines[0]
5055 error_line = lines[first_error]
5056 error_msg = lines[first_error+1:]
5057 error_msg_s = "\n".join(error_msg) + "\n"
5058 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5060 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5061 units = [simplejson.loads(line) for line in lines[:first_error]]
5062 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5063 last_unit = units[-1]
5064 self.failUnlessEqual(last_unit["path"], ["subdir"])
5065 d.addCallback(_check_broken_manifest)
5067 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5068 def _check_broken_deepcheck(res):
5069 lines = res.splitlines()
5071 for (i,line) in enumerate(lines)
5072 if line.startswith("ERROR:")]
5074 self.fail("no ERROR: in output: %s" % (res,))
5075 first_error = error_lines[0]
5076 error_line = lines[first_error]
5077 error_msg = lines[first_error+1:]
5078 error_msg_s = "\n".join(error_msg) + "\n"
5079 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5081 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5082 units = [simplejson.loads(line) for line in lines[:first_error]]
5083 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5084 last_unit = units[-1]
5085 self.failUnlessEqual(last_unit["path"], ["subdir"])
5086 r = last_unit["check-results"]["results"]
5087 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5088 self.failUnlessReallyEqual(r["count-shares-good"], 1)
5089 self.failUnlessReallyEqual(r["recoverable"], False)
5090 d.addCallback(_check_broken_deepcheck)
5092 d.addErrback(self.explain_web_error)
5095 def test_deep_check_and_repair(self):
5096 self.basedir = "web/Grid/deep_check_and_repair"
5098 c0 = self.g.clients[0]
5102 d = c0.create_dirnode()
5103 def _stash_root_and_create_file(n):
5105 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5106 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5107 d.addCallback(_stash_root_and_create_file)
5108 def _stash_uri(fn, which):
5109 self.uris[which] = fn.get_uri()
5110 d.addCallback(_stash_uri, "good")
5111 d.addCallback(lambda ign:
5112 self.rootnode.add_file(u"small",
5113 upload.Data("literal",
5115 d.addCallback(_stash_uri, "small")
5116 d.addCallback(lambda ign:
5117 self.rootnode.add_file(u"sick",
5118 upload.Data(DATA+"1",
5120 d.addCallback(_stash_uri, "sick")
5121 #d.addCallback(lambda ign:
5122 # self.rootnode.add_file(u"dead",
5123 # upload.Data(DATA+"2",
5125 #d.addCallback(_stash_uri, "dead")
5127 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5128 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5129 #d.addCallback(_stash_uri, "corrupt")
5131 def _clobber_shares(ignored):
5132 good_shares = self.find_uri_shares(self.uris["good"])
5133 self.failUnlessReallyEqual(len(good_shares), 10)
5134 sick_shares = self.find_uri_shares(self.uris["sick"])
5135 os.unlink(sick_shares[0][2])
5136 #dead_shares = self.find_uri_shares(self.uris["dead"])
5137 #for i in range(1, 10):
5138 # os.unlink(dead_shares[i][2])
5140 #c_shares = self.find_uri_shares(self.uris["corrupt"])
5141 #cso = CorruptShareOptions()
5142 #cso.stdout = StringIO()
5143 #cso.parseOptions([c_shares[0][2]])
5145 d.addCallback(_clobber_shares)
5148 # root/good CHK, 10 shares
5150 # root/sick CHK, 9 shares
5152 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5154 units = [simplejson.loads(line)
5155 for line in res.splitlines()
5157 self.failUnlessReallyEqual(len(units), 4+1)
5158 # should be parent-first
5160 self.failUnlessEqual(u0["path"], [])
5161 self.failUnlessEqual(u0["type"], "directory")
5162 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5163 u0crr = u0["check-and-repair-results"]
5164 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5165 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5167 ugood = [u for u in units
5168 if u["type"] == "file" and u["path"] == [u"good"]][0]
5169 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5170 ugoodcrr = ugood["check-and-repair-results"]
5171 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5172 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5174 usick = [u for u in units
5175 if u["type"] == "file" and u["path"] == [u"sick"]][0]
5176 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5177 usickcrr = usick["check-and-repair-results"]
5178 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5179 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5180 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5181 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5184 self.failUnlessEqual(stats["type"], "stats")
5186 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5187 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5188 self.failUnlessReallyEqual(s["count-directories"], 1)
5189 d.addCallback(_done)
5191 d.addErrback(self.explain_web_error)
5194 def _count_leases(self, ignored, which):
5195 u = self.uris[which]
5196 shares = self.find_uri_shares(u)
5198 for shnum, serverid, fn in shares:
5199 sf = get_share_file(fn)
5200 num_leases = len(list(sf.get_leases()))
5201 lease_counts.append( (fn, num_leases) )
5204 def _assert_leasecount(self, lease_counts, expected):
5205 for (fn, num_leases) in lease_counts:
5206 if num_leases != expected:
5207 self.fail("expected %d leases, have %d, on %s" %
5208 (expected, num_leases, fn))
5210 def test_add_lease(self):
5211 self.basedir = "web/Grid/add_lease"
5212 self.set_up_grid(num_clients=2)
5213 c0 = self.g.clients[0]
5216 d = c0.upload(upload.Data(DATA, convergence=""))
5217 def _stash_uri(ur, which):
5218 self.uris[which] = ur.get_uri()
5219 d.addCallback(_stash_uri, "one")
5220 d.addCallback(lambda ign:
5221 c0.upload(upload.Data(DATA+"1", convergence="")))
5222 d.addCallback(_stash_uri, "two")
5223 def _stash_mutable_uri(n, which):
5224 self.uris[which] = n.get_uri()
5225 assert isinstance(self.uris[which], str)
5226 d.addCallback(lambda ign:
5227 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5228 d.addCallback(_stash_mutable_uri, "mutable")
5230 def _compute_fileurls(ignored):
5232 for which in self.uris:
5233 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5234 d.addCallback(_compute_fileurls)
5236 d.addCallback(self._count_leases, "one")
5237 d.addCallback(self._assert_leasecount, 1)
5238 d.addCallback(self._count_leases, "two")
5239 d.addCallback(self._assert_leasecount, 1)
5240 d.addCallback(self._count_leases, "mutable")
5241 d.addCallback(self._assert_leasecount, 1)
5243 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5244 def _got_html_good(res):
5245 self.failUnlessIn("Healthy", res)
5246 self.failIfIn("Not Healthy", res)
5247 d.addCallback(_got_html_good)
5249 d.addCallback(self._count_leases, "one")
5250 d.addCallback(self._assert_leasecount, 1)
5251 d.addCallback(self._count_leases, "two")
5252 d.addCallback(self._assert_leasecount, 1)
5253 d.addCallback(self._count_leases, "mutable")
5254 d.addCallback(self._assert_leasecount, 1)
5256 # this CHECK uses the original client, which uses the same
5257 # lease-secrets, so it will just renew the original lease
5258 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5259 d.addCallback(_got_html_good)
5261 d.addCallback(self._count_leases, "one")
5262 d.addCallback(self._assert_leasecount, 1)
5263 d.addCallback(self._count_leases, "two")
5264 d.addCallback(self._assert_leasecount, 1)
5265 d.addCallback(self._count_leases, "mutable")
5266 d.addCallback(self._assert_leasecount, 1)
5268 # this CHECK uses an alternate client, which adds a second lease
5269 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5270 d.addCallback(_got_html_good)
5272 d.addCallback(self._count_leases, "one")
5273 d.addCallback(self._assert_leasecount, 2)
5274 d.addCallback(self._count_leases, "two")
5275 d.addCallback(self._assert_leasecount, 1)
5276 d.addCallback(self._count_leases, "mutable")
5277 d.addCallback(self._assert_leasecount, 1)
5279 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5280 d.addCallback(_got_html_good)
5282 d.addCallback(self._count_leases, "one")
5283 d.addCallback(self._assert_leasecount, 2)
5284 d.addCallback(self._count_leases, "two")
5285 d.addCallback(self._assert_leasecount, 1)
5286 d.addCallback(self._count_leases, "mutable")
5287 d.addCallback(self._assert_leasecount, 1)
5289 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5291 d.addCallback(_got_html_good)
5293 d.addCallback(self._count_leases, "one")
5294 d.addCallback(self._assert_leasecount, 2)
5295 d.addCallback(self._count_leases, "two")
5296 d.addCallback(self._assert_leasecount, 1)
5297 d.addCallback(self._count_leases, "mutable")
5298 d.addCallback(self._assert_leasecount, 2)
5300 d.addErrback(self.explain_web_error)
5303 def test_deep_add_lease(self):
5304 self.basedir = "web/Grid/deep_add_lease"
5305 self.set_up_grid(num_clients=2)
5306 c0 = self.g.clients[0]
5310 d = c0.create_dirnode()
5311 def _stash_root_and_create_file(n):
5313 self.uris["root"] = n.get_uri()
5314 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5315 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5316 d.addCallback(_stash_root_and_create_file)
5317 def _stash_uri(fn, which):
5318 self.uris[which] = fn.get_uri()
5319 d.addCallback(_stash_uri, "one")
5320 d.addCallback(lambda ign:
5321 self.rootnode.add_file(u"small",
5322 upload.Data("literal",
5324 d.addCallback(_stash_uri, "small")
5326 d.addCallback(lambda ign:
5327 c0.create_mutable_file(publish.MutableData("mutable")))
5328 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5329 d.addCallback(_stash_uri, "mutable")
5331 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5333 units = [simplejson.loads(line)
5334 for line in res.splitlines()
5336 # root, one, small, mutable, stats
5337 self.failUnlessReallyEqual(len(units), 4+1)
5338 d.addCallback(_done)
5340 d.addCallback(self._count_leases, "root")
5341 d.addCallback(self._assert_leasecount, 1)
5342 d.addCallback(self._count_leases, "one")
5343 d.addCallback(self._assert_leasecount, 1)
5344 d.addCallback(self._count_leases, "mutable")
5345 d.addCallback(self._assert_leasecount, 1)
5347 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5348 d.addCallback(_done)
5350 d.addCallback(self._count_leases, "root")
5351 d.addCallback(self._assert_leasecount, 1)
5352 d.addCallback(self._count_leases, "one")
5353 d.addCallback(self._assert_leasecount, 1)
5354 d.addCallback(self._count_leases, "mutable")
5355 d.addCallback(self._assert_leasecount, 1)
5357 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5359 d.addCallback(_done)
5361 d.addCallback(self._count_leases, "root")
5362 d.addCallback(self._assert_leasecount, 2)
5363 d.addCallback(self._count_leases, "one")
5364 d.addCallback(self._assert_leasecount, 2)
5365 d.addCallback(self._count_leases, "mutable")
5366 d.addCallback(self._assert_leasecount, 2)
5368 d.addErrback(self.explain_web_error)
5372 def test_exceptions(self):
5373 self.basedir = "web/Grid/exceptions"
5374 self.set_up_grid(num_clients=1, num_servers=2)
5375 c0 = self.g.clients[0]
5376 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5379 d = c0.create_dirnode()
5381 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5382 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5384 d.addCallback(_stash_root)
5385 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5387 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5388 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5390 u = uri.from_string(ur.get_uri())
5391 u.key = testutil.flip_bit(u.key, 0)
5392 baduri = u.to_string()
5393 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5394 d.addCallback(_stash_bad)
5395 d.addCallback(lambda ign: c0.create_dirnode())
5396 def _mangle_dirnode_1share(n):
5398 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5399 self.fileurls["dir-1share-json"] = url + "?t=json"
5400 self.delete_shares_numbered(u, range(1,10))
5401 d.addCallback(_mangle_dirnode_1share)
5402 d.addCallback(lambda ign: c0.create_dirnode())
5403 def _mangle_dirnode_0share(n):
5405 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5406 self.fileurls["dir-0share-json"] = url + "?t=json"
5407 self.delete_shares_numbered(u, range(0,10))
5408 d.addCallback(_mangle_dirnode_0share)
5410 # NotEnoughSharesError should be reported sensibly, with a
5411 # text/plain explanation of the problem, and perhaps some
5412 # information on which shares *could* be found.
5414 d.addCallback(lambda ignored:
5415 self.shouldHTTPError("GET unrecoverable",
5416 410, "Gone", "NoSharesError",
5417 self.GET, self.fileurls["0shares"]))
5418 def _check_zero_shares(body):
5419 self.failIfIn("<html>", body)
5420 body = " ".join(body.strip().split())
5421 exp = ("NoSharesError: no shares could be found. "
5422 "Zero shares usually indicates a corrupt URI, or that "
5423 "no servers were connected, but it might also indicate "
5424 "severe corruption. You should perform a filecheck on "
5425 "this object to learn more. The full error message is: "
5426 "no shares (need 3). Last failure: None")
5427 self.failUnlessReallyEqual(exp, body)
5428 d.addCallback(_check_zero_shares)
5431 d.addCallback(lambda ignored:
5432 self.shouldHTTPError("GET 1share",
5433 410, "Gone", "NotEnoughSharesError",
5434 self.GET, self.fileurls["1share"]))
5435 def _check_one_share(body):
5436 self.failIfIn("<html>", body)
5437 body = " ".join(body.strip().split())
5438 msgbase = ("NotEnoughSharesError: This indicates that some "
5439 "servers were unavailable, or that shares have been "
5440 "lost to server departure, hard drive failure, or disk "
5441 "corruption. You should perform a filecheck on "
5442 "this object to learn more. The full error message is:"
5444 msg1 = msgbase + (" ran out of shares:"
5447 " overdue= unused= need 3. Last failure: None")
5448 msg2 = msgbase + (" ran out of shares:"
5450 " pending=Share(sh0-on-xgru5)"
5451 " overdue= unused= need 3. Last failure: None")
5452 self.failUnless(body == msg1 or body == msg2, body)
5453 d.addCallback(_check_one_share)
5455 d.addCallback(lambda ignored:
5456 self.shouldHTTPError("GET imaginary",
5457 404, "Not Found", None,
5458 self.GET, self.fileurls["imaginary"]))
5459 def _missing_child(body):
5460 self.failUnlessIn("No such child: imaginary", body)
5461 d.addCallback(_missing_child)
5463 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5464 def _check_0shares_dir_html(body):
5465 self.failUnlessIn("<html>", body)
5466 # we should see the regular page, but without the child table or
5468 body = " ".join(body.strip().split())
5469 self.failUnlessIn('href="?t=info">More info on this directory',
5471 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5472 "could not be retrieved, because there were insufficient "
5473 "good shares. This might indicate that no servers were "
5474 "connected, insufficient servers were connected, the URI "
5475 "was corrupt, or that shares have been lost due to server "
5476 "departure, hard drive failure, or disk corruption. You "
5477 "should perform a filecheck on this object to learn more.")
5478 self.failUnlessIn(exp, body)
5479 self.failUnlessIn("No upload forms: directory is unreadable", body)
5480 d.addCallback(_check_0shares_dir_html)
5482 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5483 def _check_1shares_dir_html(body):
5484 # at some point, we'll split UnrecoverableFileError into 0-shares
5485 # and some-shares like we did for immutable files (since there
5486 # are different sorts of advice to offer in each case). For now,
5487 # they present the same way.
5488 self.failUnlessIn("<html>", body)
5489 body = " ".join(body.strip().split())
5490 self.failUnlessIn('href="?t=info">More info on this directory',
5492 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5493 "could not be retrieved, because there were insufficient "
5494 "good shares. This might indicate that no servers were "
5495 "connected, insufficient servers were connected, the URI "
5496 "was corrupt, or that shares have been lost due to server "
5497 "departure, hard drive failure, or disk corruption. You "
5498 "should perform a filecheck on this object to learn more.")
5499 self.failUnlessIn(exp, body)
5500 self.failUnlessIn("No upload forms: directory is unreadable", body)
5501 d.addCallback(_check_1shares_dir_html)
5503 d.addCallback(lambda ignored:
5504 self.shouldHTTPError("GET dir-0share-json",
5505 410, "Gone", "UnrecoverableFileError",
5507 self.fileurls["dir-0share-json"]))
5508 def _check_unrecoverable_file(body):
5509 self.failIfIn("<html>", body)
5510 body = " ".join(body.strip().split())
5511 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5512 "could not be retrieved, because there were insufficient "
5513 "good shares. This might indicate that no servers were "
5514 "connected, insufficient servers were connected, the URI "
5515 "was corrupt, or that shares have been lost due to server "
5516 "departure, hard drive failure, or disk corruption. You "
5517 "should perform a filecheck on this object to learn more.")
5518 self.failUnlessReallyEqual(exp, body)
5519 d.addCallback(_check_unrecoverable_file)
5521 d.addCallback(lambda ignored:
5522 self.shouldHTTPError("GET dir-1share-json",
5523 410, "Gone", "UnrecoverableFileError",
5525 self.fileurls["dir-1share-json"]))
5526 d.addCallback(_check_unrecoverable_file)
5528 d.addCallback(lambda ignored:
5529 self.shouldHTTPError("GET imaginary",
5530 404, "Not Found", None,
5531 self.GET, self.fileurls["imaginary"]))
5533 # attach a webapi child that throws a random error, to test how it
5535 w = c0.getServiceNamed("webish")
5536 w.root.putChild("ERRORBOOM", ErrorBoom())
5538 # "Accept: */*" : should get a text/html stack trace
5539 # "Accept: text/plain" : should get a text/plain stack trace
5540 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5541 # no Accept header: should get a text/html stack trace
5543 d.addCallback(lambda ignored:
5544 self.shouldHTTPError("GET errorboom_html",
5545 500, "Internal Server Error", None,
5546 self.GET, "ERRORBOOM",
5547 headers={"accept": "*/*"}))
5548 def _internal_error_html1(body):
5549 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5550 d.addCallback(_internal_error_html1)
5552 d.addCallback(lambda ignored:
5553 self.shouldHTTPError("GET errorboom_text",
5554 500, "Internal Server Error", None,
5555 self.GET, "ERRORBOOM",
5556 headers={"accept": "text/plain"}))
5557 def _internal_error_text2(body):
5558 self.failIfIn("<html>", body)
5559 self.failUnless(body.startswith("Traceback "), body)
5560 d.addCallback(_internal_error_text2)
5562 CLI_accepts = "text/plain, application/octet-stream"
5563 d.addCallback(lambda ignored:
5564 self.shouldHTTPError("GET errorboom_text",
5565 500, "Internal Server Error", None,
5566 self.GET, "ERRORBOOM",
5567 headers={"accept": CLI_accepts}))
5568 def _internal_error_text3(body):
5569 self.failIfIn("<html>", body)
5570 self.failUnless(body.startswith("Traceback "), body)
5571 d.addCallback(_internal_error_text3)
5573 d.addCallback(lambda ignored:
5574 self.shouldHTTPError("GET errorboom_text",
5575 500, "Internal Server Error", None,
5576 self.GET, "ERRORBOOM"))
5577 def _internal_error_html4(body):
5578 self.failUnlessIn("<html>", body)
5579 d.addCallback(_internal_error_html4)
5581 def _flush_errors(res):
5582 # Trial: please ignore the CompletelyUnhandledError in the logs
5583 self.flushLoggedErrors(CompletelyUnhandledError)
5585 d.addBoth(_flush_errors)
5589 def test_blacklist(self):
5590 # download from a blacklisted URI, get an error
5591 self.basedir = "web/Grid/blacklist"
5593 c0 = self.g.clients[0]
5594 c0_basedir = c0.basedir
5595 fn = os.path.join(c0_basedir, "access.blacklist")
5597 DATA = "off-limits " * 50
5599 d = c0.upload(upload.Data(DATA, convergence=""))
5600 def _stash_uri_and_create_dir(ur):
5601 self.uri = ur.get_uri()
5602 self.url = "uri/"+self.uri
5603 u = uri.from_string_filenode(self.uri)
5604 self.si = u.get_storage_index()
5605 childnode = c0.create_node_from_uri(self.uri, None)
5606 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5607 d.addCallback(_stash_uri_and_create_dir)
5608 def _stash_dir(node):
5609 self.dir_node = node
5610 self.dir_uri = node.get_uri()
5611 self.dir_url = "uri/"+self.dir_uri
5612 d.addCallback(_stash_dir)
5613 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5614 def _check_dir_html(body):
5615 self.failUnlessIn("<html>", body)
5616 self.failUnlessIn("blacklisted.txt</a>", body)
5617 d.addCallback(_check_dir_html)
5618 d.addCallback(lambda ign: self.GET(self.url))
5619 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5621 def _blacklist(ign):
5623 f.write(" # this is a comment\n")
5625 f.write("\n") # also exercise blank lines
5626 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5628 # clients should be checking the blacklist each time, so we don't
5629 # need to restart the client
5630 d.addCallback(_blacklist)
5631 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5633 "Access Prohibited: off-limits",
5634 self.GET, self.url))
5636 # We should still be able to list the parent directory, in HTML...
5637 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5638 def _check_dir_html2(body):
5639 self.failUnlessIn("<html>", body)
5640 self.failUnlessIn("blacklisted.txt</strike>", body)
5641 d.addCallback(_check_dir_html2)
5643 # ... and in JSON (used by CLI).
5644 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5645 def _check_dir_json(res):
5646 data = simplejson.loads(res)
5647 self.failUnless(isinstance(data, list), data)
5648 self.failUnlessEqual(data[0], "dirnode")
5649 self.failUnless(isinstance(data[1], dict), data)
5650 self.failUnlessIn("children", data[1])
5651 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5652 childdata = data[1]["children"]["blacklisted.txt"]
5653 self.failUnless(isinstance(childdata, list), data)
5654 self.failUnlessEqual(childdata[0], "filenode")
5655 self.failUnless(isinstance(childdata[1], dict), data)
5656 d.addCallback(_check_dir_json)
5658 def _unblacklist(ign):
5659 open(fn, "w").close()
5660 # the Blacklist object watches mtime to tell when the file has
5661 # changed, but on windows this test will run faster than the
5662 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5663 # to force a reload.
5664 self.g.clients[0].blacklist.last_mtime -= 2.0
5665 d.addCallback(_unblacklist)
5667 # now a read should work
5668 d.addCallback(lambda ign: self.GET(self.url))
5669 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5671 # read again to exercise the blacklist-is-unchanged logic
5672 d.addCallback(lambda ign: self.GET(self.url))
5673 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5675 # now add a blacklisted directory, and make sure files under it are
5678 childnode = c0.create_node_from_uri(self.uri, None)
5679 return c0.create_dirnode({u"child": (childnode,{}) })
5680 d.addCallback(_add_dir)
5681 def _get_dircap(dn):
5682 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5683 self.dir_url_base = "uri/"+dn.get_write_uri()
5684 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5685 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5686 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5687 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5688 d.addCallback(_get_dircap)
5689 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5690 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5691 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5692 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5693 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5694 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5695 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5696 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5697 d.addCallback(lambda ign: self.GET(self.child_url))
5698 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5700 def _block_dir(ign):
5702 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5704 self.g.clients[0].blacklist.last_mtime -= 2.0
5705 d.addCallback(_block_dir)
5706 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5708 "Access Prohibited: dir-off-limits",
5709 self.GET, self.dir_url_base))
5710 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5712 "Access Prohibited: dir-off-limits",
5713 self.GET, self.dir_url_json1))
5714 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5716 "Access Prohibited: dir-off-limits",
5717 self.GET, self.dir_url_json2))
5718 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5720 "Access Prohibited: dir-off-limits",
5721 self.GET, self.dir_url_json_ro))
5722 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5724 "Access Prohibited: dir-off-limits",
5725 self.GET, self.child_url))
5729 class CompletelyUnhandledError(Exception):
5731 class ErrorBoom(rend.Page):
5732 def beforeRender(self, ctx):
5733 raise CompletelyUnhandledError("whoops")