1 import os.path, re, urllib, time, cgi
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow.util import escapeToXML
15 from nevow import rend
17 from allmydata import interfaces, uri, webish, dirnode
18 from allmydata.storage.shares import get_share_file
19 from allmydata.storage_client import StorageFarmBroker, StubServer
20 from allmydata.immutable import upload
21 from allmydata.immutable.downloader.status import DownloadStatus
22 from allmydata.dirnode import DirectoryNode
23 from allmydata.nodemaker import NodeMaker
24 from allmydata.unknown import UnknownNode
25 from allmydata.web import status, common
26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
27 from allmydata.util import fileutil, base32, hashutil
28 from allmydata.util.consumer import download_to_data
29 from allmydata.util.netstring import split_netstring
30 from allmydata.util.encodingutil import to_str
31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
32 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
33 make_mutable_file_uri, create_mutable_filenode
34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
35 from allmydata.mutable import servermap, publish, retrieve
36 import allmydata.test.common_util as testutil
37 from allmydata.test.no_network import GridTestMixin
38 from allmydata.test.common_web import HTTPClientGETFactory, \
40 from allmydata.client import Client, SecretHolder
41 from allmydata.introducer import IntroducerNode
43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
44 # create a webserver that works against them
46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
55 class FakeStatsProvider:
57 stats = {'stats': {}, 'counters': {}}
60 class FakeNodeMaker(NodeMaker):
65 'max_segment_size':128*1024 # 1024=KiB
67 def _create_lit(self, cap):
68 return FakeCHKFileNode(cap, self.all_contents)
69 def _create_immutable(self, cap):
70 return FakeCHKFileNode(cap, self.all_contents)
71 def _create_mutable(self, cap):
72 return FakeMutableFileNode(None, None,
73 self.encoding_params, None,
74 self.all_contents).init_from_cap(cap)
75 def create_mutable_file(self, contents="", keysize=None,
76 version=SDMF_VERSION):
77 n = FakeMutableFileNode(None, None, self.encoding_params, None,
79 return n.create(contents, version=version)
81 class FakeUploader(service.Service):
84 helper_connected = False
86 def upload(self, uploadable):
87 d = uploadable.get_size()
88 d.addCallback(lambda size: uploadable.read(size))
91 n = create_chk_filenode(data, self.all_contents)
92 ur = upload.UploadResults(file_size=len(data),
99 uri_extension_data={},
100 uri_extension_hash="fake",
101 verifycapstr="fakevcap")
102 ur.set_uri(n.get_uri())
104 d.addCallback(_got_data)
107 def get_helper_info(self):
108 return (self.helper_furl, self.helper_connected)
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeDisplayableServer(StubServer):
175 def __init__(self, serverid, nickname):
176 StubServer.__init__(self, serverid)
177 self.announcement = {"my-version": "allmydata-tahoe-fake",
178 "service-name": "storage",
179 "nickname": nickname}
180 def is_connected(self):
182 def get_permutation_seed(self):
184 def get_remote_host(self):
186 def get_last_loss_time(self):
188 def get_announcement_time(self):
190 def get_announcement(self):
191 return self.announcement
192 def get_nickname(self):
193 return self.announcement["nickname"]
195 class FakeBucketCounter(object):
197 return {"last-complete-bucket-count": 0}
198 def get_progress(self):
199 return {"estimated-time-per-cycle": 0,
200 "cycle-in-progress": False,
201 "remaining-wait-time": 0}
203 class FakeLeaseChecker(object):
205 self.expiration_enabled = False
207 self.override_lease_duration = None
208 self.sharetypes_to_expire = {}
210 return {"history": None}
211 def get_progress(self):
212 return {"estimated-time-per-cycle": 0,
213 "cycle-in-progress": False,
214 "remaining-wait-time": 0}
216 class FakeStorageServer(service.MultiService):
218 def __init__(self, nodeid, nickname):
219 service.MultiService.__init__(self)
220 self.my_nodeid = nodeid
221 self.nickname = nickname
222 self.bucket_counter = FakeBucketCounter()
223 self.lease_checker = FakeLeaseChecker()
225 return {"storage_server.accepting_immutable_shares": False}
227 class FakeClient(Client):
229 # don't upcall to Client.__init__, since we only want to initialize a
231 service.MultiService.__init__(self)
232 self.all_contents = {}
233 self.nodeid = "fake_nodeid"
234 self.nickname = u"fake_nickname \u263A"
235 self.introducer_furl = "None"
236 self.stats_provider = FakeStatsProvider()
237 self._secret_holder = SecretHolder("lease secret", "convergence secret")
239 self.convergence = "some random string"
240 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
241 # fake knowledge of another server
242 self.storage_broker.test_add_server("other_nodeid",
243 FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
244 self.introducer_client = None
245 self.history = FakeHistory()
246 self.uploader = FakeUploader()
247 self.uploader.all_contents = self.all_contents
248 self.uploader.setServiceParent(self)
249 self.blacklist = None
250 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
253 self.nodemaker.all_contents = self.all_contents
254 self.mutable_file_default = SDMF_VERSION
255 self.addService(FakeStorageServer(self.nodeid, self.nickname))
257 def get_long_nodeid(self):
259 def get_long_tubid(self):
262 def startService(self):
263 return service.MultiService.startService(self)
264 def stopService(self):
265 return service.MultiService.stopService(self)
267 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
269 class WebMixin(object):
271 self.s = FakeClient()
272 self.s.startService()
273 self.staticdir = self.mktemp()
275 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
277 self.ws.setServiceParent(self.s)
278 self.webish_port = self.ws.getPortnum()
279 self.webish_url = self.ws.getURL()
280 assert self.webish_url.endswith("/")
281 self.webish_url = self.webish_url[:-1] # these tests add their own /
283 l = [ self.s.create_dirnode() for x in range(6) ]
284 d = defer.DeferredList(l)
286 self.public_root = res[0][1]
287 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
288 self.public_url = "/uri/" + self.public_root.get_uri()
289 self.private_root = res[1][1]
293 self._foo_uri = foo.get_uri()
294 self._foo_readonly_uri = foo.get_readonly_uri()
295 self._foo_verifycap = foo.get_verify_cap().to_string()
296 # NOTE: we ignore the deferred on all set_uri() calls, because we
297 # know the fake nodes do these synchronously
298 self.public_root.set_uri(u"foo", foo.get_uri(),
299 foo.get_readonly_uri())
301 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
302 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
303 self._bar_txt_verifycap = n.get_verify_cap().to_string()
306 # XXX: Do we ever use this?
307 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
309 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
312 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
313 assert self._quux_txt_uri.startswith("URI:MDMF")
314 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
316 foo.set_uri(u"empty", res[3][1].get_uri(),
317 res[3][1].get_readonly_uri())
318 sub_uri = res[4][1].get_uri()
319 self._sub_uri = sub_uri
320 foo.set_uri(u"sub", sub_uri, sub_uri)
321 sub = self.s.create_node_from_uri(sub_uri)
324 _ign, n, blocking_uri = self.makefile(1)
325 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
327 # filenode to test for html encoding issues
328 self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
329 self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
330 self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
331 self._htmlname_escaped = escapeToXML(self._htmlname_raw)
332 self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
333 self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
334 self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
335 foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
337 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
338 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
339 # still think of it as an umlaut
340 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
342 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
343 self._baz_file_uri = baz_file
344 sub.set_uri(u"baz.txt", baz_file, baz_file)
346 _ign, n, self._bad_file_uri = self.makefile(3)
347 # this uri should not be downloadable
348 del self.s.all_contents[self._bad_file_uri]
351 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
352 rodir.get_readonly_uri())
353 rodir.set_uri(u"nor", baz_file, baz_file)
359 # public/foo/quux.txt
360 # public/foo/blockingfile
361 # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
364 # public/foo/sub/baz.txt
366 # public/reedownlee/nor
367 self.NEWFILE_CONTENTS = "newfile contents\n"
369 return foo.get_metadata_for(u"bar.txt")
371 def _got_metadata(metadata):
372 self._bar_txt_metadata = metadata
373 d.addCallback(_got_metadata)
376 def get_all_contents(self):
377 return self.s.all_contents
379 def makefile(self, number):
380 contents = "contents of file %s\n" % number
381 n = create_chk_filenode(contents, self.get_all_contents())
382 return contents, n, n.get_uri()
384 def makefile_mutable(self, number, mdmf=False):
385 contents = "contents of mutable file %s\n" % number
386 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
387 return contents, n, n.get_uri(), n.get_readonly_uri()
390 return self.s.stopService()
392 def failUnlessIsBarDotTxt(self, res):
393 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
395 def failUnlessIsQuuxDotTxt(self, res):
396 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
398 def failUnlessIsBazDotTxt(self, res):
399 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
401 def failUnlessIsSubBazDotTxt(self, res):
402 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
404 def failUnlessIsBarJSON(self, res):
405 data = simplejson.loads(res)
406 self.failUnless(isinstance(data, list))
407 self.failUnlessEqual(data[0], "filenode")
408 self.failUnless(isinstance(data[1], dict))
409 self.failIf(data[1]["mutable"])
410 self.failIfIn("rw_uri", data[1]) # immutable
411 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
412 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
413 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
415 def failUnlessIsQuuxJSON(self, res, readonly=False):
416 data = simplejson.loads(res)
417 self.failUnless(isinstance(data, list))
418 self.failUnlessEqual(data[0], "filenode")
419 self.failUnless(isinstance(data[1], dict))
421 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
423 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
424 self.failUnless(metadata['mutable'])
426 self.failIfIn("rw_uri", metadata)
428 self.failUnlessIn("rw_uri", metadata)
429 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
430 self.failUnlessIn("ro_uri", metadata)
431 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
432 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
434 def failUnlessIsFooJSON(self, res):
435 data = simplejson.loads(res)
436 self.failUnless(isinstance(data, list))
437 self.failUnlessEqual(data[0], "dirnode", res)
438 self.failUnless(isinstance(data[1], dict))
439 self.failUnless(data[1]["mutable"])
440 self.failUnlessIn("rw_uri", data[1]) # mutable
441 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
442 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
443 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
445 kidnames = sorted([unicode(n) for n in data[1]["children"]])
446 self.failUnlessEqual(kidnames,
447 [self._htmlname_unicode, u"bar.txt", u"baz.txt",
448 u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
449 kids = dict( [(unicode(name),value)
451 in data[1]["children"].iteritems()] )
452 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
453 self.failUnlessIn("metadata", kids[u"sub"][1])
454 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
455 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
456 self.failUnlessIn("linkcrtime", tahoe_md)
457 self.failUnlessIn("linkmotime", tahoe_md)
458 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
459 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
460 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
461 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
462 self._bar_txt_verifycap)
463 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
464 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
465 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
466 self._bar_txt_metadata["tahoe"]["linkcrtime"])
467 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
469 self.failUnlessIn("quux.txt", kids)
470 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
472 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
473 self._quux_txt_readonly_uri)
475 def GET(self, urlpath, followRedirect=False, return_response=False,
477 # if return_response=True, this fires with (data, statuscode,
478 # respheaders) instead of just data.
479 assert not isinstance(urlpath, unicode)
480 url = self.webish_url + urlpath
481 factory = HTTPClientGETFactory(url, method="GET",
482 followRedirect=followRedirect, **kwargs)
483 reactor.connectTCP("localhost", self.webish_port, factory)
486 return (data, factory.status, factory.response_headers)
488 d.addCallback(_got_data)
489 return factory.deferred
491 def HEAD(self, urlpath, return_response=False, **kwargs):
492 # this requires some surgery, because twisted.web.client doesn't want
493 # to give us back the response headers.
494 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
495 reactor.connectTCP("localhost", self.webish_port, factory)
498 return (data, factory.status, factory.response_headers)
500 d.addCallback(_got_data)
501 return factory.deferred
503 def PUT(self, urlpath, data, **kwargs):
504 url = self.webish_url + urlpath
505 return client.getPage(url, method="PUT", postdata=data, **kwargs)
507 def DELETE(self, urlpath):
508 url = self.webish_url + urlpath
509 return client.getPage(url, method="DELETE")
511 def POST(self, urlpath, followRedirect=False, **fields):
512 sepbase = "boogabooga"
516 form.append('Content-Disposition: form-data; name="_charset"')
520 for name, value in fields.iteritems():
521 if isinstance(value, tuple):
522 filename, value = value
523 form.append('Content-Disposition: form-data; name="%s"; '
524 'filename="%s"' % (name, filename.encode("utf-8")))
526 form.append('Content-Disposition: form-data; name="%s"' % name)
528 if isinstance(value, unicode):
529 value = value.encode("utf-8")
532 assert isinstance(value, str)
539 body = "\r\n".join(form) + "\r\n"
540 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
541 return self.POST2(urlpath, body, headers, followRedirect)
543 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
544 url = self.webish_url + urlpath
545 return client.getPage(url, method="POST", postdata=body,
546 headers=headers, followRedirect=followRedirect)
548 def shouldFail(self, res, expected_failure, which,
549 substring=None, response_substring=None):
550 if isinstance(res, failure.Failure):
551 res.trap(expected_failure)
553 self.failUnlessIn(substring, str(res), which)
554 if response_substring:
555 self.failUnlessIn(response_substring, res.value.response, which)
557 self.fail("%s was supposed to raise %s, not get '%s'" %
558 (which, expected_failure, res))
560 def shouldFail2(self, expected_failure, which, substring,
562 callable, *args, **kwargs):
563 assert substring is None or isinstance(substring, str)
564 assert response_substring is None or isinstance(response_substring, str)
565 d = defer.maybeDeferred(callable, *args, **kwargs)
567 if isinstance(res, failure.Failure):
568 res.trap(expected_failure)
570 self.failUnlessIn(substring, str(res),
571 "'%s' not in '%s' (response is '%s') for test '%s'" % \
572 (substring, str(res),
573 getattr(res.value, "response", ""),
575 if response_substring:
576 self.failUnlessIn(response_substring, res.value.response,
577 "'%s' not in '%s' for test '%s'" % \
578 (response_substring, res.value.response,
581 self.fail("%s was supposed to raise %s, not get '%s'" %
582 (which, expected_failure, res))
586 def should404(self, res, which):
587 if isinstance(res, failure.Failure):
588 res.trap(error.Error)
589 self.failUnlessReallyEqual(res.value.status, "404")
591 self.fail("%s was supposed to Error(404), not get '%s'" %
594 def should302(self, res, which):
595 if isinstance(res, failure.Failure):
596 res.trap(error.Error)
597 self.failUnlessReallyEqual(res.value.status, "302")
599 self.fail("%s was supposed to Error(302), not get '%s'" %
603 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
604 def test_create(self):
607 def test_welcome(self):
610 self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
611 self.failUnlessIn(FAVICON_MARKUP, res)
612 self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
613 self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
614 self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
615 res_u = res.decode('utf-8')
616 self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
617 self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
618 self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
620 self.s.basedir = 'web/test_welcome'
621 fileutil.make_dirs("web/test_welcome")
622 fileutil.make_dirs("web/test_welcome/private")
624 d.addCallback(_check)
627 def test_introducer_status(self):
628 class MockIntroducerClient(object):
629 def __init__(self, connected):
630 self.connected = connected
631 def connected_to_introducer(self):
632 return self.connected
634 d = defer.succeed(None)
636 # introducer not connected, unguessable furl
637 def _set_introducer_not_connected_unguessable(ign):
638 self.s.introducer_furl = "pb://someIntroducer/secret"
639 self.s.introducer_client = MockIntroducerClient(False)
641 d.addCallback(_set_introducer_not_connected_unguessable)
642 def _check_introducer_not_connected_unguessable(res):
643 html = res.replace('\n', ' ')
644 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
645 self.failIfIn('pb://someIntroducer/secret', html)
646 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
647 d.addCallback(_check_introducer_not_connected_unguessable)
649 # introducer connected, unguessable furl
650 def _set_introducer_connected_unguessable(ign):
651 self.s.introducer_furl = "pb://someIntroducer/secret"
652 self.s.introducer_client = MockIntroducerClient(True)
654 d.addCallback(_set_introducer_connected_unguessable)
655 def _check_introducer_connected_unguessable(res):
656 html = res.replace('\n', ' ')
657 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
658 self.failIfIn('pb://someIntroducer/secret', html)
659 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
660 d.addCallback(_check_introducer_connected_unguessable)
662 # introducer connected, guessable furl
663 def _set_introducer_connected_guessable(ign):
664 self.s.introducer_furl = "pb://someIntroducer/introducer"
665 self.s.introducer_client = MockIntroducerClient(True)
667 d.addCallback(_set_introducer_connected_guessable)
668 def _check_introducer_connected_guessable(res):
669 html = res.replace('\n', ' ')
670 self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
671 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
672 d.addCallback(_check_introducer_connected_guessable)
675 def test_helper_status(self):
676 d = defer.succeed(None)
678 # set helper furl to None
679 def _set_no_helper(ign):
680 self.s.uploader.helper_furl = None
682 d.addCallback(_set_no_helper)
683 def _check_no_helper(res):
684 html = res.replace('\n', ' ')
685 self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
686 d.addCallback(_check_no_helper)
688 # enable helper, not connected
689 def _set_helper_not_connected(ign):
690 self.s.uploader.helper_furl = "pb://someHelper/secret"
691 self.s.uploader.helper_connected = False
693 d.addCallback(_set_helper_not_connected)
694 def _check_helper_not_connected(res):
695 html = res.replace('\n', ' ')
696 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
697 self.failIfIn('pb://someHelper/secret', html)
698 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
699 d.addCallback(_check_helper_not_connected)
701 # enable helper, connected
702 def _set_helper_connected(ign):
703 self.s.uploader.helper_furl = "pb://someHelper/secret"
704 self.s.uploader.helper_connected = True
706 d.addCallback(_set_helper_connected)
707 def _check_helper_connected(res):
708 html = res.replace('\n', ' ')
709 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
710 self.failIfIn('pb://someHelper/secret', html)
711 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
712 d.addCallback(_check_helper_connected)
715 def test_storage(self):
716 d = self.GET("/storage")
718 self.failUnlessIn('Storage Server Status', res)
719 self.failUnlessIn(FAVICON_MARKUP, res)
720 res_u = res.decode('utf-8')
721 self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
722 d.addCallback(_check)
725 def test_status(self):
726 h = self.s.get_history()
727 dl_num = h.list_all_download_statuses()[0].get_counter()
728 ul_num = h.list_all_upload_statuses()[0].get_counter()
729 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
730 pub_num = h.list_all_publish_statuses()[0].get_counter()
731 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
732 d = self.GET("/status", followRedirect=True)
734 self.failUnlessIn('Recent and Active Operations', res)
735 self.failUnlessIn('"down-%d"' % dl_num, res)
736 self.failUnlessIn('"up-%d"' % ul_num, res)
737 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
738 self.failUnlessIn('"publish-%d"' % pub_num, res)
739 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
740 d.addCallback(_check)
741 d.addCallback(lambda res: self.GET("/status/?t=json"))
742 def _check_json(res):
743 data = simplejson.loads(res)
744 self.failUnless(isinstance(data, dict))
745 #active = data["active"]
746 # TODO: test more. We need a way to fake an active operation
748 d.addCallback(_check_json)
750 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
752 self.failUnlessIn("File Download Status", res)
753 d.addCallback(_check_dl)
754 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
755 def _check_dl_json(res):
756 data = simplejson.loads(res)
757 self.failUnless(isinstance(data, dict))
758 self.failUnlessIn("read", data)
759 self.failUnlessEqual(data["read"][0]["length"], 120)
760 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
761 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
762 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
763 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
764 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
765 # serverids[] keys are strings, since that's what JSON does, but
766 # we'd really like them to be ints
767 self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
768 self.failUnless(data["serverids"].has_key("1"),
769 str(data["serverids"]))
770 self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
771 str(data["serverids"]))
772 self.failUnlessEqual(data["server_info"][phwr_id]["short"],
774 self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
776 self.failUnlessIn("dyhb", data)
777 self.failUnlessIn("misc", data)
778 d.addCallback(_check_dl_json)
779 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
781 self.failUnlessIn("File Upload Status", res)
782 d.addCallback(_check_ul)
783 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
784 def _check_mapupdate(res):
785 self.failUnlessIn("Mutable File Servermap Update Status", res)
786 d.addCallback(_check_mapupdate)
787 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
788 def _check_publish(res):
789 self.failUnlessIn("Mutable File Publish Status", res)
790 d.addCallback(_check_publish)
791 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
792 def _check_retrieve(res):
793 self.failUnlessIn("Mutable File Retrieve Status", res)
794 d.addCallback(_check_retrieve)
798 def test_status_numbers(self):
799 drrm = status.DownloadResultsRendererMixin()
800 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
801 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
802 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
803 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
804 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
805 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
806 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
807 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
808 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
810 urrm = status.UploadResultsRendererMixin()
811 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
812 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
813 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
814 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
815 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
816 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
817 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
818 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
819 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
821 def test_GET_FILEURL(self):
822 d = self.GET(self.public_url + "/foo/bar.txt")
823 d.addCallback(self.failUnlessIsBarDotTxt)
826 def test_GET_FILEURL_range(self):
827 headers = {"range": "bytes=1-10"}
828 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
829 return_response=True)
830 def _got((res, status, headers)):
831 self.failUnlessReallyEqual(int(status), 206)
832 self.failUnless(headers.has_key("content-range"))
833 self.failUnlessReallyEqual(headers["content-range"][0],
834 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
835 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
839 def test_GET_FILEURL_partial_range(self):
840 headers = {"range": "bytes=5-"}
841 length = len(self.BAR_CONTENTS)
842 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
843 return_response=True)
844 def _got((res, status, headers)):
845 self.failUnlessReallyEqual(int(status), 206)
846 self.failUnless(headers.has_key("content-range"))
847 self.failUnlessReallyEqual(headers["content-range"][0],
848 "bytes 5-%d/%d" % (length-1, length))
849 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
853 def test_GET_FILEURL_partial_end_range(self):
854 headers = {"range": "bytes=-5"}
855 length = len(self.BAR_CONTENTS)
856 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
857 return_response=True)
858 def _got((res, status, headers)):
859 self.failUnlessReallyEqual(int(status), 206)
860 self.failUnless(headers.has_key("content-range"))
861 self.failUnlessReallyEqual(headers["content-range"][0],
862 "bytes %d-%d/%d" % (length-5, length-1, length))
863 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
867 def test_GET_FILEURL_partial_range_overrun(self):
868 headers = {"range": "bytes=100-200"}
869 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
870 "416 Requested Range not satisfiable",
871 "First beyond end of file",
872 self.GET, self.public_url + "/foo/bar.txt",
876 def test_HEAD_FILEURL_range(self):
877 headers = {"range": "bytes=1-10"}
878 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
879 return_response=True)
880 def _got((res, status, headers)):
881 self.failUnlessReallyEqual(res, "")
882 self.failUnlessReallyEqual(int(status), 206)
883 self.failUnless(headers.has_key("content-range"))
884 self.failUnlessReallyEqual(headers["content-range"][0],
885 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
889 def test_HEAD_FILEURL_partial_range(self):
890 headers = {"range": "bytes=5-"}
891 length = len(self.BAR_CONTENTS)
892 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
893 return_response=True)
894 def _got((res, status, headers)):
895 self.failUnlessReallyEqual(int(status), 206)
896 self.failUnless(headers.has_key("content-range"))
897 self.failUnlessReallyEqual(headers["content-range"][0],
898 "bytes 5-%d/%d" % (length-1, length))
902 def test_HEAD_FILEURL_partial_end_range(self):
903 headers = {"range": "bytes=-5"}
904 length = len(self.BAR_CONTENTS)
905 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
906 return_response=True)
907 def _got((res, status, headers)):
908 self.failUnlessReallyEqual(int(status), 206)
909 self.failUnless(headers.has_key("content-range"))
910 self.failUnlessReallyEqual(headers["content-range"][0],
911 "bytes %d-%d/%d" % (length-5, length-1, length))
915 def test_HEAD_FILEURL_partial_range_overrun(self):
916 headers = {"range": "bytes=100-200"}
917 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
918 "416 Requested Range not satisfiable",
920 self.HEAD, self.public_url + "/foo/bar.txt",
924 def test_GET_FILEURL_range_bad(self):
925 headers = {"range": "BOGUS=fizbop-quarnak"}
926 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
927 return_response=True)
928 def _got((res, status, headers)):
929 self.failUnlessReallyEqual(int(status), 200)
930 self.failUnless(not headers.has_key("content-range"))
931 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
935 def test_HEAD_FILEURL(self):
936 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
937 def _got((res, status, headers)):
938 self.failUnlessReallyEqual(res, "")
939 self.failUnlessReallyEqual(headers["content-length"][0],
940 str(len(self.BAR_CONTENTS)))
941 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
945 def test_GET_FILEURL_named(self):
946 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
947 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
948 d = self.GET(base + "/@@name=/blah.txt")
949 d.addCallback(self.failUnlessIsBarDotTxt)
950 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
951 d.addCallback(self.failUnlessIsBarDotTxt)
952 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
953 d.addCallback(self.failUnlessIsBarDotTxt)
954 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
955 d.addCallback(self.failUnlessIsBarDotTxt)
956 save_url = base + "?save=true&filename=blah.txt"
957 d.addCallback(lambda res: self.GET(save_url))
958 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
959 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
960 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
961 u_url = base + "?save=true&filename=" + u_fn_e
962 d.addCallback(lambda res: self.GET(u_url))
963 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
966 def test_PUT_FILEURL_named_bad(self):
967 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
968 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
970 "/file can only be used with GET or HEAD",
971 self.PUT, base + "/@@name=/blah.txt", "")
975 def test_GET_DIRURL_named_bad(self):
976 base = "/file/%s" % urllib.quote(self._foo_uri)
977 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
980 self.GET, base + "/@@name=/blah.txt")
983 def test_GET_slash_file_bad(self):
984 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
986 "/file must be followed by a file-cap and a name",
990 def test_GET_unhandled_URI_named(self):
991 contents, n, newuri = self.makefile(12)
992 verifier_cap = n.get_verify_cap().to_string()
993 base = "/file/%s" % urllib.quote(verifier_cap)
994 # client.create_node_from_uri() can't handle verify-caps
995 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
996 "400 Bad Request", "is not a file-cap",
1000 def test_GET_unhandled_URI(self):
1001 contents, n, newuri = self.makefile(12)
1002 verifier_cap = n.get_verify_cap().to_string()
1003 base = "/uri/%s" % urllib.quote(verifier_cap)
1004 # client.create_node_from_uri() can't handle verify-caps
1005 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1007 "GET unknown URI type: can only do t=info",
1011 def test_GET_FILE_URI(self):
1012 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1014 d.addCallback(self.failUnlessIsBarDotTxt)
1017 def test_GET_FILE_URI_mdmf(self):
1018 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1020 d.addCallback(self.failUnlessIsQuuxDotTxt)
1023 def test_GET_FILE_URI_mdmf_extensions(self):
1024 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1026 d.addCallback(self.failUnlessIsQuuxDotTxt)
1029 def test_GET_FILE_URI_mdmf_readonly(self):
1030 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1032 d.addCallback(self.failUnlessIsQuuxDotTxt)
1035 def test_GET_FILE_URI_badchild(self):
1036 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1037 errmsg = "Files have no children, certainly not named 'boguschild'"
1038 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1039 "400 Bad Request", errmsg,
1043 def test_PUT_FILE_URI_badchild(self):
1044 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1045 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1046 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1047 "400 Bad Request", errmsg,
1051 def test_PUT_FILE_URI_mdmf(self):
1052 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1053 self._quux_new_contents = "new_contents"
1055 d.addCallback(lambda res:
1056 self.failUnlessIsQuuxDotTxt(res))
1057 d.addCallback(lambda ignored:
1058 self.PUT(base, self._quux_new_contents))
1059 d.addCallback(lambda ignored:
1061 d.addCallback(lambda res:
1062 self.failUnlessReallyEqual(res, self._quux_new_contents))
1065 def test_PUT_FILE_URI_mdmf_extensions(self):
1066 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1067 self._quux_new_contents = "new_contents"
1069 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1070 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1071 d.addCallback(lambda ignored: self.GET(base))
1072 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1076 def test_PUT_FILE_URI_mdmf_readonly(self):
1077 # We're not allowed to PUT things to a readonly cap.
1078 base = "/uri/%s" % self._quux_txt_readonly_uri
1080 d.addCallback(lambda res:
1081 self.failUnlessIsQuuxDotTxt(res))
1082 # What should we get here? We get a 500 error now; that's not right.
1083 d.addCallback(lambda ignored:
1084 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1085 "400 Bad Request", "read-only cap",
1086 self.PUT, base, "new data"))
1089 def test_PUT_FILE_URI_sdmf_readonly(self):
1090 # We're not allowed to put things to a readonly cap.
1091 base = "/uri/%s" % self._baz_txt_readonly_uri
1093 d.addCallback(lambda res:
1094 self.failUnlessIsBazDotTxt(res))
1095 d.addCallback(lambda ignored:
1096 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1097 "400 Bad Request", "read-only cap",
1098 self.PUT, base, "new_data"))
1101 def test_GET_etags(self):
1103 def _check_etags(uri):
1105 d2 = _get_etag(uri, 'json')
1106 d = defer.DeferredList([d1, d2], consumeErrors=True)
1107 def _check(results):
1108 # All deferred must succeed
1109 self.failUnless(all([r[0] for r in results]))
1110 # the etag for the t=json form should be just like the etag
1111 # fo the default t='' form, but with a 'json' suffix
1112 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1113 d.addCallback(_check)
1116 def _get_etag(uri, t=''):
1117 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1118 d = self.GET(targetbase, return_response=True, followRedirect=True)
1119 def _just_the_etag(result):
1120 data, response, headers = result
1121 etag = headers['etag'][0]
1122 if uri.startswith('URI:DIR'):
1123 self.failUnless(etag.startswith('DIR:'), etag)
1125 return d.addCallback(_just_the_etag)
1127 # Check that etags work with immutable directories
1128 (newkids, caps) = self._create_immutable_children()
1129 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1130 simplejson.dumps(newkids))
1131 def _stash_immdir_uri(uri):
1132 self._immdir_uri = uri
1134 d.addCallback(_stash_immdir_uri)
1135 d.addCallback(_check_etags)
1137 # Check that etags work with immutable files
1138 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1140 # use the ETag on GET
1141 def _check_match(ign):
1142 uri = "/uri/%s" % self._bar_txt_uri
1143 d = self.GET(uri, return_response=True)
1145 d.addCallback(lambda (data, code, headers):
1147 # do a GET that's supposed to match the ETag
1148 d.addCallback(lambda etag:
1149 self.GET(uri, return_response=True,
1150 headers={"If-None-Match": etag}))
1151 # make sure it short-circuited (304 instead of 200)
1152 d.addCallback(lambda (data, code, headers):
1153 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1155 d.addCallback(_check_match)
1157 def _no_etag(uri, t):
1158 target = "/uri/%s?t=%s" % (uri, t)
1159 d = self.GET(target, return_response=True, followRedirect=True)
1160 d.addCallback(lambda (data, code, headers):
1161 self.failIf("etag" in headers, target))
1163 def _yes_etag(uri, t):
1164 target = "/uri/%s?t=%s" % (uri, t)
1165 d = self.GET(target, return_response=True, followRedirect=True)
1166 d.addCallback(lambda (data, code, headers):
1167 self.failUnless("etag" in headers, target))
1170 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1171 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1172 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1173 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1174 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1176 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1177 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1178 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1179 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1180 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1181 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1185 # TODO: version of this with a Unicode filename
1186 def test_GET_FILEURL_save(self):
1187 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1188 return_response=True)
1189 def _got((res, statuscode, headers)):
1190 content_disposition = headers["content-disposition"][0]
1191 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1192 self.failUnlessIsBarDotTxt(res)
1196 def test_GET_FILEURL_missing(self):
1197 d = self.GET(self.public_url + "/foo/missing")
1198 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1201 def test_GET_FILEURL_info_mdmf(self):
1202 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1204 self.failUnlessIn("mutable file (mdmf)", res)
1205 self.failUnlessIn(self._quux_txt_uri, res)
1206 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1210 def test_GET_FILEURL_info_mdmf_readonly(self):
1211 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1213 self.failUnlessIn("mutable file (mdmf)", res)
1214 self.failIfIn(self._quux_txt_uri, res)
1215 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1219 def test_GET_FILEURL_info_sdmf(self):
1220 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1222 self.failUnlessIn("mutable file (sdmf)", res)
1223 self.failUnlessIn(self._baz_txt_uri, res)
1227 def test_GET_FILEURL_info_mdmf_extensions(self):
1228 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1230 self.failUnlessIn("mutable file (mdmf)", res)
1231 self.failUnlessIn(self._quux_txt_uri, res)
1232 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1236 def test_PUT_overwrite_only_files(self):
1237 # create a directory, put a file in that directory.
1238 contents, n, filecap = self.makefile(8)
1239 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1240 d.addCallback(lambda res:
1241 self.PUT(self.public_url + "/foo/dir/file1.txt",
1242 self.NEWFILE_CONTENTS))
1243 # try to overwrite the file with replace=only-files
1244 # (this should work)
1245 d.addCallback(lambda res:
1246 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1248 d.addCallback(lambda res:
1249 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1250 "There was already a child by that name, and you asked me "
1251 "to not replace it",
1252 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1256 def test_PUT_NEWFILEURL(self):
1257 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1258 # TODO: we lose the response code, so we can't check this
1259 #self.failUnlessReallyEqual(responsecode, 201)
1260 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1261 d.addCallback(lambda res:
1262 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1263 self.NEWFILE_CONTENTS))
1266 def test_PUT_NEWFILEURL_not_mutable(self):
1267 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1268 self.NEWFILE_CONTENTS)
1269 # TODO: we lose the response code, so we can't check this
1270 #self.failUnlessReallyEqual(responsecode, 201)
1271 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1272 d.addCallback(lambda res:
1273 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1274 self.NEWFILE_CONTENTS))
1277 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1278 # this should get us a few segments of an MDMF mutable file,
1279 # which we can then test for.
1280 contents = self.NEWFILE_CONTENTS * 300000
1281 d = self.PUT("/uri?format=mdmf",
1283 def _got_filecap(filecap):
1284 self.failUnless(filecap.startswith("URI:MDMF"))
1286 d.addCallback(_got_filecap)
1287 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1288 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1291 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1292 contents = self.NEWFILE_CONTENTS * 300000
1293 d = self.PUT("/uri?format=sdmf",
1295 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1296 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1299 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1300 contents = self.NEWFILE_CONTENTS * 300000
1301 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1302 400, "Bad Request", "Unknown format: foo",
1303 self.PUT, "/uri?format=foo",
1306 def test_PUT_NEWFILEURL_range_bad(self):
1307 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1308 target = self.public_url + "/foo/new.txt"
1309 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1310 "501 Not Implemented",
1311 "Content-Range in PUT not yet supported",
1312 # (and certainly not for immutable files)
1313 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1315 d.addCallback(lambda res:
1316 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1319 def test_PUT_NEWFILEURL_mutable(self):
1320 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1321 self.NEWFILE_CONTENTS)
1322 # TODO: we lose the response code, so we can't check this
1323 #self.failUnlessReallyEqual(responsecode, 201)
1324 def _check_uri(res):
1325 u = uri.from_string_mutable_filenode(res)
1326 self.failUnless(u.is_mutable())
1327 self.failIf(u.is_readonly())
1329 d.addCallback(_check_uri)
1330 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1331 d.addCallback(lambda res:
1332 self.failUnlessMutableChildContentsAre(self._foo_node,
1334 self.NEWFILE_CONTENTS))
1337 def test_PUT_NEWFILEURL_mutable_toobig(self):
1338 # It is okay to upload large mutable files, so we should be able
1340 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1341 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1344 def test_PUT_NEWFILEURL_replace(self):
1345 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1346 # TODO: we lose the response code, so we can't check this
1347 #self.failUnlessReallyEqual(responsecode, 200)
1348 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1349 d.addCallback(lambda res:
1350 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1351 self.NEWFILE_CONTENTS))
1354 def test_PUT_NEWFILEURL_bad_t(self):
1355 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1356 "PUT to a file: bad t=bogus",
1357 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1361 def test_PUT_NEWFILEURL_no_replace(self):
1362 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1363 self.NEWFILE_CONTENTS)
1364 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1366 "There was already a child by that name, and you asked me "
1367 "to not replace it")
1370 def test_PUT_NEWFILEURL_mkdirs(self):
1371 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1373 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1374 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1375 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1376 d.addCallback(lambda res:
1377 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1378 self.NEWFILE_CONTENTS))
1381 def test_PUT_NEWFILEURL_blocked(self):
1382 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1383 self.NEWFILE_CONTENTS)
1384 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1386 "Unable to create directory 'blockingfile': a file was in the way")
1389 def test_PUT_NEWFILEURL_emptyname(self):
1390 # an empty pathname component (i.e. a double-slash) is disallowed
1391 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1393 "The webapi does not allow empty pathname components",
1394 self.PUT, self.public_url + "/foo//new.txt", "")
1397 def test_DELETE_FILEURL(self):
1398 d = self.DELETE(self.public_url + "/foo/bar.txt")
1399 d.addCallback(lambda res:
1400 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1403 def test_DELETE_FILEURL_missing(self):
1404 d = self.DELETE(self.public_url + "/foo/missing")
1405 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1408 def test_DELETE_FILEURL_missing2(self):
1409 d = self.DELETE(self.public_url + "/missing/missing")
1410 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1413 def failUnlessHasBarDotTxtMetadata(self, res):
1414 data = simplejson.loads(res)
1415 self.failUnless(isinstance(data, list))
1416 self.failUnlessIn("metadata", data[1])
1417 self.failUnlessIn("tahoe", data[1]["metadata"])
1418 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1419 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1420 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1421 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1423 def test_GET_FILEURL_json(self):
1424 # twisted.web.http.parse_qs ignores any query args without an '=', so
1425 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1426 # instead. This may make it tricky to emulate the S3 interface
1428 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1430 self.failUnlessIsBarJSON(data)
1431 self.failUnlessHasBarDotTxtMetadata(data)
1433 d.addCallback(_check1)
1436 def test_GET_FILEURL_json_mutable_type(self):
1437 # The JSON should include format, which says whether the
1438 # file is SDMF or MDMF
1439 d = self.PUT("/uri?format=mdmf",
1440 self.NEWFILE_CONTENTS * 300000)
1441 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1442 def _got_json(json, version):
1443 data = simplejson.loads(json)
1444 assert "filenode" == data[0]
1446 assert isinstance(data, dict)
1448 self.failUnlessIn("format", data)
1449 self.failUnlessEqual(data["format"], version)
1451 d.addCallback(_got_json, "MDMF")
1452 # Now make an SDMF file and check that it is reported correctly.
1453 d.addCallback(lambda ignored:
1454 self.PUT("/uri?format=sdmf",
1455 self.NEWFILE_CONTENTS * 300000))
1456 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1457 d.addCallback(_got_json, "SDMF")
1460 def test_GET_FILEURL_json_mdmf(self):
1461 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1462 d.addCallback(self.failUnlessIsQuuxJSON)
1465 def test_GET_FILEURL_json_missing(self):
1466 d = self.GET(self.public_url + "/foo/missing?json")
1467 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1470 def test_GET_FILEURL_uri(self):
1471 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1473 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1474 d.addCallback(_check)
1475 d.addCallback(lambda res:
1476 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1478 # for now, for files, uris and readonly-uris are the same
1479 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1480 d.addCallback(_check2)
1483 def test_GET_FILEURL_badtype(self):
1484 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1487 self.public_url + "/foo/bar.txt?t=bogus")
1490 def test_CSS_FILE(self):
1491 d = self.GET("/tahoe.css", followRedirect=True)
1493 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1494 self.failUnless(CSS_STYLE.search(res), res)
1495 d.addCallback(_check)
1498 def test_GET_FILEURL_uri_missing(self):
1499 d = self.GET(self.public_url + "/foo/missing?t=uri")
1500 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1503 def _check_upload_and_mkdir_forms(self, html):
1504 # We should have a form to create a file, with radio buttons that allow
1505 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1506 self.failUnlessIn('name="t" value="upload"', html)
1507 self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1508 self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1509 self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1511 # We should also have the ability to create a mutable directory, with
1512 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1513 # or MDMF directory.
1514 self.failUnlessIn('name="t" value="mkdir"', html)
1515 self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1516 self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1518 self.failUnlessIn(FAVICON_MARKUP, html)
1520 def test_GET_DIRECTORY_html(self):
1521 d = self.GET(self.public_url + "/foo", followRedirect=True)
1523 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1524 self._check_upload_and_mkdir_forms(html)
1525 self.failUnlessIn("quux", html)
1526 d.addCallback(_check)
1529 def test_GET_DIRECTORY_html_filenode_encoding(self):
1530 d = self.GET(self.public_url + "/foo", followRedirect=True)
1532 # Check if encoded entries are there
1533 self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1534 + self._htmlname_escaped + '</a>', html)
1535 self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1536 self.failIfIn(self._htmlname_escaped_double, html)
1537 # Make sure that Nevow escaping actually works by checking for unsafe characters
1538 # and that '&' is escaped.
1540 self.failUnlessIn(entity, self._htmlname_raw)
1541 self.failIfIn(entity, self._htmlname_escaped)
1542 self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1543 self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1544 d.addCallback(_check)
1547 def test_GET_root_html(self):
1549 d.addCallback(self._check_upload_and_mkdir_forms)
1552 def test_GET_DIRURL(self):
1553 # the addSlash means we get a redirect here
1554 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1556 d = self.GET(self.public_url + "/foo", followRedirect=True)
1558 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1560 # the FILE reference points to a URI, but it should end in bar.txt
1561 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1562 (ROOT, urllib.quote(self._bar_txt_uri)))
1563 get_bar = "".join([r'<td>FILE</td>',
1565 r'<a href="%s">bar.txt</a>' % bar_url,
1567 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1569 self.failUnless(re.search(get_bar, res), res)
1570 for label in ['unlink', 'rename/relink']:
1571 for line in res.split("\n"):
1572 # find the line that contains the relevant button for bar.txt
1573 if ("form action" in line and
1574 ('value="%s"' % (label,)) in line and
1575 'value="bar.txt"' in line):
1576 # the form target should use a relative URL
1577 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1578 self.failUnlessIn('action="%s"' % foo_url, line)
1579 # and the when_done= should too
1580 #done_url = urllib.quote(???)
1581 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1583 # 'unlink' needs to use POST because it directly has a side effect
1584 if label == 'unlink':
1585 self.failUnlessIn('method="post"', line)
1588 self.fail("unable to find '%s bar.txt' line" % (label,))
1590 # the DIR reference just points to a URI
1591 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1592 get_sub = ((r'<td>DIR</td>')
1593 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1594 self.failUnless(re.search(get_sub, res), res)
1595 d.addCallback(_check)
1597 # look at a readonly directory
1598 d.addCallback(lambda res:
1599 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1601 self.failUnlessIn("(read-only)", res)
1602 self.failIfIn("Upload a file", res)
1603 d.addCallback(_check2)
1605 # and at a directory that contains a readonly directory
1606 d.addCallback(lambda res:
1607 self.GET(self.public_url, followRedirect=True))
1609 self.failUnless(re.search('<td>DIR-RO</td>'
1610 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1611 d.addCallback(_check3)
1613 # and an empty directory
1614 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1616 self.failUnlessIn("directory is empty", res)
1617 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1618 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1619 d.addCallback(_check4)
1621 # and at a literal directory
1622 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1623 d.addCallback(lambda res:
1624 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1626 self.failUnlessIn('(immutable)', res)
1627 self.failUnless(re.search('<td>FILE</td>'
1628 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1629 d.addCallback(_check5)
1632 def test_GET_DIRURL_badtype(self):
1633 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1637 self.public_url + "/foo?t=bogus")
1640 def test_GET_DIRURL_json(self):
1641 d = self.GET(self.public_url + "/foo?t=json")
1642 d.addCallback(self.failUnlessIsFooJSON)
1645 def test_GET_DIRURL_json_format(self):
1646 d = self.PUT(self.public_url + \
1647 "/foo/sdmf.txt?format=sdmf",
1648 self.NEWFILE_CONTENTS * 300000)
1649 d.addCallback(lambda ignored:
1650 self.PUT(self.public_url + \
1651 "/foo/mdmf.txt?format=mdmf",
1652 self.NEWFILE_CONTENTS * 300000))
1653 # Now we have an MDMF and SDMF file in the directory. If we GET
1654 # its JSON, we should see their encodings.
1655 d.addCallback(lambda ignored:
1656 self.GET(self.public_url + "/foo?t=json"))
1657 def _got_json(json):
1658 data = simplejson.loads(json)
1659 assert data[0] == "dirnode"
1662 kids = data['children']
1664 mdmf_data = kids['mdmf.txt'][1]
1665 self.failUnlessIn("format", mdmf_data)
1666 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1668 sdmf_data = kids['sdmf.txt'][1]
1669 self.failUnlessIn("format", sdmf_data)
1670 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1671 d.addCallback(_got_json)
1675 def test_POST_DIRURL_manifest_no_ophandle(self):
1676 d = self.shouldFail2(error.Error,
1677 "test_POST_DIRURL_manifest_no_ophandle",
1679 "slow operation requires ophandle=",
1680 self.POST, self.public_url, t="start-manifest")
1683 def test_POST_DIRURL_manifest(self):
1684 d = defer.succeed(None)
1685 def getman(ignored, output):
1686 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1687 followRedirect=True)
1688 d.addCallback(self.wait_for_operation, "125")
1689 d.addCallback(self.get_operation_results, "125", output)
1691 d.addCallback(getman, None)
1692 def _got_html(manifest):
1693 self.failUnlessIn("Manifest of SI=", manifest)
1694 self.failUnlessIn("<td>sub</td>", manifest)
1695 self.failUnlessIn(self._sub_uri, manifest)
1696 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1697 self.failUnlessIn(FAVICON_MARKUP, manifest)
1698 d.addCallback(_got_html)
1700 # both t=status and unadorned GET should be identical
1701 d.addCallback(lambda res: self.GET("/operations/125"))
1702 d.addCallback(_got_html)
1704 d.addCallback(getman, "html")
1705 d.addCallback(_got_html)
1706 d.addCallback(getman, "text")
1707 def _got_text(manifest):
1708 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1709 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1710 d.addCallback(_got_text)
1711 d.addCallback(getman, "JSON")
1713 data = res["manifest"]
1715 for (path_list, cap) in data:
1716 got[tuple(path_list)] = cap
1717 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1718 self.failUnlessIn((u"sub", u"baz.txt"), got)
1719 self.failUnlessIn("finished", res)
1720 self.failUnlessIn("origin", res)
1721 self.failUnlessIn("storage-index", res)
1722 self.failUnlessIn("verifycaps", res)
1723 self.failUnlessIn("stats", res)
1724 d.addCallback(_got_json)
1727 def test_POST_DIRURL_deepsize_no_ophandle(self):
1728 d = self.shouldFail2(error.Error,
1729 "test_POST_DIRURL_deepsize_no_ophandle",
1731 "slow operation requires ophandle=",
1732 self.POST, self.public_url, t="start-deep-size")
1735 def test_POST_DIRURL_deepsize(self):
1736 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1737 followRedirect=True)
1738 d.addCallback(self.wait_for_operation, "126")
1739 d.addCallback(self.get_operation_results, "126", "json")
1740 def _got_json(data):
1741 self.failUnlessReallyEqual(data["finished"], True)
1743 self.failUnless(size > 1000)
1744 d.addCallback(_got_json)
1745 d.addCallback(self.get_operation_results, "126", "text")
1747 mo = re.search(r'^size: (\d+)$', res, re.M)
1748 self.failUnless(mo, res)
1749 size = int(mo.group(1))
1750 # with directories, the size varies.
1751 self.failUnless(size > 1000)
1752 d.addCallback(_got_text)
1755 def test_POST_DIRURL_deepstats_no_ophandle(self):
1756 d = self.shouldFail2(error.Error,
1757 "test_POST_DIRURL_deepstats_no_ophandle",
1759 "slow operation requires ophandle=",
1760 self.POST, self.public_url, t="start-deep-stats")
1763 def test_POST_DIRURL_deepstats(self):
1764 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1765 followRedirect=True)
1766 d.addCallback(self.wait_for_operation, "127")
1767 d.addCallback(self.get_operation_results, "127", "json")
1768 def _got_json(stats):
1769 expected = {"count-immutable-files": 4,
1770 "count-mutable-files": 2,
1771 "count-literal-files": 0,
1773 "count-directories": 3,
1774 "size-immutable-files": 76,
1775 "size-literal-files": 0,
1776 #"size-directories": 1912, # varies
1777 #"largest-directory": 1590,
1778 "largest-directory-children": 8,
1779 "largest-immutable-file": 19,
1781 for k,v in expected.iteritems():
1782 self.failUnlessReallyEqual(stats[k], v,
1783 "stats[%s] was %s, not %s" %
1785 self.failUnlessReallyEqual(stats["size-files-histogram"],
1787 d.addCallback(_got_json)
1790 def test_POST_DIRURL_stream_manifest(self):
1791 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1793 self.failUnless(res.endswith("\n"))
1794 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1795 self.failUnlessReallyEqual(len(units), 10)
1796 self.failUnlessEqual(units[-1]["type"], "stats")
1798 self.failUnlessEqual(first["path"], [])
1799 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1800 self.failUnlessEqual(first["type"], "directory")
1801 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1802 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1803 self.failIfEqual(baz["storage-index"], None)
1804 self.failIfEqual(baz["verifycap"], None)
1805 self.failIfEqual(baz["repaircap"], None)
1806 # XXX: Add quux and baz to this test.
1808 d.addCallback(_check)
1811 def test_GET_DIRURL_uri(self):
1812 d = self.GET(self.public_url + "/foo?t=uri")
1814 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1815 d.addCallback(_check)
1818 def test_GET_DIRURL_readonly_uri(self):
1819 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1821 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1822 d.addCallback(_check)
1825 def test_PUT_NEWDIRURL(self):
1826 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1827 d.addCallback(lambda res:
1828 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1829 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1830 d.addCallback(self.failUnlessNodeKeysAre, [])
1833 def test_PUT_NEWDIRURL_mdmf(self):
1834 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1835 d.addCallback(lambda res:
1836 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1837 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1838 d.addCallback(lambda node:
1839 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1842 def test_PUT_NEWDIRURL_sdmf(self):
1843 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1845 d.addCallback(lambda res:
1846 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1847 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1848 d.addCallback(lambda node:
1849 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1852 def test_PUT_NEWDIRURL_bad_format(self):
1853 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1854 400, "Bad Request", "Unknown format: foo",
1855 self.PUT, self.public_url +
1856 "/foo/newdir=?t=mkdir&format=foo", "")
1858 def test_POST_NEWDIRURL(self):
1859 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1860 d.addCallback(lambda res:
1861 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1862 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1863 d.addCallback(self.failUnlessNodeKeysAre, [])
1866 def test_POST_NEWDIRURL_mdmf(self):
1867 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1868 d.addCallback(lambda res:
1869 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1870 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1871 d.addCallback(lambda node:
1872 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1875 def test_POST_NEWDIRURL_sdmf(self):
1876 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1877 d.addCallback(lambda res:
1878 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1879 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1880 d.addCallback(lambda node:
1881 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1884 def test_POST_NEWDIRURL_bad_format(self):
1885 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1886 400, "Bad Request", "Unknown format: foo",
1887 self.POST2, self.public_url + \
1888 "/foo/newdir?t=mkdir&format=foo", "")
1890 def test_POST_NEWDIRURL_emptyname(self):
1891 # an empty pathname component (i.e. a double-slash) is disallowed
1892 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1894 "The webapi does not allow empty pathname components, i.e. a double slash",
1895 self.POST, self.public_url + "//?t=mkdir")
1898 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1899 (newkids, caps) = self._create_initial_children()
1900 query = "/foo/newdir?t=mkdir-with-children"
1901 if version == MDMF_VERSION:
1902 query += "&format=mdmf"
1903 elif version == SDMF_VERSION:
1904 query += "&format=sdmf"
1906 version = SDMF_VERSION # for later
1907 d = self.POST2(self.public_url + query,
1908 simplejson.dumps(newkids))
1910 n = self.s.create_node_from_uri(uri.strip())
1911 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1912 self.failUnlessEqual(n._node.get_version(), version)
1913 d2.addCallback(lambda ign:
1914 self.failUnlessROChildURIIs(n, u"child-imm",
1916 d2.addCallback(lambda ign:
1917 self.failUnlessRWChildURIIs(n, u"child-mutable",
1919 d2.addCallback(lambda ign:
1920 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1922 d2.addCallback(lambda ign:
1923 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1924 caps['unknown_rocap']))
1925 d2.addCallback(lambda ign:
1926 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1927 caps['unknown_rwcap']))
1928 d2.addCallback(lambda ign:
1929 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1930 caps['unknown_immcap']))
1931 d2.addCallback(lambda ign:
1932 self.failUnlessRWChildURIIs(n, u"dirchild",
1934 d2.addCallback(lambda ign:
1935 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1937 d2.addCallback(lambda ign:
1938 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1939 caps['emptydircap']))
1941 d.addCallback(_check)
1942 d.addCallback(lambda res:
1943 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1944 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1945 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1946 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1947 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1950 def test_POST_NEWDIRURL_initial_children(self):
1951 return self._do_POST_NEWDIRURL_initial_children_test()
1953 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1954 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1956 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1957 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1959 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1960 (newkids, caps) = self._create_initial_children()
1961 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1962 400, "Bad Request", "Unknown format: foo",
1963 self.POST2, self.public_url + \
1964 "/foo/newdir?t=mkdir-with-children&format=foo",
1965 simplejson.dumps(newkids))
1967 def test_POST_NEWDIRURL_immutable(self):
1968 (newkids, caps) = self._create_immutable_children()
1969 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1970 simplejson.dumps(newkids))
1972 n = self.s.create_node_from_uri(uri.strip())
1973 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1974 d2.addCallback(lambda ign:
1975 self.failUnlessROChildURIIs(n, u"child-imm",
1977 d2.addCallback(lambda ign:
1978 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1979 caps['unknown_immcap']))
1980 d2.addCallback(lambda ign:
1981 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1983 d2.addCallback(lambda ign:
1984 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1986 d2.addCallback(lambda ign:
1987 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1988 caps['emptydircap']))
1990 d.addCallback(_check)
1991 d.addCallback(lambda res:
1992 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1993 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1994 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1995 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1996 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1997 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1998 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2005 d.addErrback(self.explain_web_error)
2008 def test_POST_NEWDIRURL_immutable_bad(self):
2009 (newkids, caps) = self._create_initial_children()
2010 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2012 "needed to be immutable but was not",
2014 self.public_url + "/foo/newdir?t=mkdir-immutable",
2015 simplejson.dumps(newkids))
2018 def test_PUT_NEWDIRURL_exists(self):
2019 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2020 d.addCallback(lambda res:
2021 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2022 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2023 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2026 def test_PUT_NEWDIRURL_blocked(self):
2027 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2028 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2030 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2031 d.addCallback(lambda res:
2032 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2033 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2034 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2037 def test_PUT_NEWDIRURL_mkdirs(self):
2038 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2039 d.addCallback(lambda res:
2040 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2041 d.addCallback(lambda res:
2042 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2043 d.addCallback(lambda res:
2044 self._foo_node.get_child_at_path(u"subdir/newdir"))
2045 d.addCallback(self.failUnlessNodeKeysAre, [])
2048 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2049 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2050 d.addCallback(lambda ignored:
2051 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2052 d.addCallback(lambda ignored:
2053 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2054 d.addCallback(lambda ignored:
2055 self._foo_node.get_child_at_path(u"subdir"))
2056 def _got_subdir(subdir):
2057 # XXX: What we want?
2058 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2059 self.failUnlessNodeHasChild(subdir, u"newdir")
2060 return subdir.get_child_at_path(u"newdir")
2061 d.addCallback(_got_subdir)
2062 d.addCallback(lambda newdir:
2063 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2066 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2067 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2068 d.addCallback(lambda ignored:
2069 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2070 d.addCallback(lambda ignored:
2071 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2072 d.addCallback(lambda ignored:
2073 self._foo_node.get_child_at_path(u"subdir"))
2074 def _got_subdir(subdir):
2075 # XXX: What we want?
2076 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2077 self.failUnlessNodeHasChild(subdir, u"newdir")
2078 return subdir.get_child_at_path(u"newdir")
2079 d.addCallback(_got_subdir)
2080 d.addCallback(lambda newdir:
2081 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2084 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2085 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2086 400, "Bad Request", "Unknown format: foo",
2087 self.PUT, self.public_url + \
2088 "/foo/subdir/newdir?t=mkdir&format=foo",
2091 def test_DELETE_DIRURL(self):
2092 d = self.DELETE(self.public_url + "/foo")
2093 d.addCallback(lambda res:
2094 self.failIfNodeHasChild(self.public_root, u"foo"))
2097 def test_DELETE_DIRURL_missing(self):
2098 d = self.DELETE(self.public_url + "/foo/missing")
2099 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2100 d.addCallback(lambda res:
2101 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2104 def test_DELETE_DIRURL_missing2(self):
2105 d = self.DELETE(self.public_url + "/missing")
2106 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2109 def dump_root(self):
2111 w = webish.DirnodeWalkerMixin()
2112 def visitor(childpath, childnode, metadata):
2114 d = w.walk(self.public_root, visitor)
2117 def failUnlessNodeKeysAre(self, node, expected_keys):
2118 for k in expected_keys:
2119 assert isinstance(k, unicode)
2121 def _check(children):
2122 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2123 d.addCallback(_check)
2125 def failUnlessNodeHasChild(self, node, name):
2126 assert isinstance(name, unicode)
2128 def _check(children):
2129 self.failUnlessIn(name, children)
2130 d.addCallback(_check)
2132 def failIfNodeHasChild(self, node, name):
2133 assert isinstance(name, unicode)
2135 def _check(children):
2136 self.failIfIn(name, children)
2137 d.addCallback(_check)
2140 def failUnlessChildContentsAre(self, node, name, expected_contents):
2141 assert isinstance(name, unicode)
2142 d = node.get_child_at_path(name)
2143 d.addCallback(lambda node: download_to_data(node))
2144 def _check(contents):
2145 self.failUnlessReallyEqual(contents, expected_contents)
2146 d.addCallback(_check)
2149 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2150 assert isinstance(name, unicode)
2151 d = node.get_child_at_path(name)
2152 d.addCallback(lambda node: node.download_best_version())
2153 def _check(contents):
2154 self.failUnlessReallyEqual(contents, expected_contents)
2155 d.addCallback(_check)
2158 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2159 assert isinstance(name, unicode)
2160 d = node.get_child_at_path(name)
2162 self.failUnless(child.is_unknown() or not child.is_readonly())
2163 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2164 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2165 expected_ro_uri = self._make_readonly(expected_uri)
2167 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2168 d.addCallback(_check)
2171 def failUnlessROChildURIIs(self, node, name, expected_uri):
2172 assert isinstance(name, unicode)
2173 d = node.get_child_at_path(name)
2175 self.failUnless(child.is_unknown() or child.is_readonly())
2176 self.failUnlessReallyEqual(child.get_write_uri(), None)
2177 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2178 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2179 d.addCallback(_check)
2182 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2183 assert isinstance(name, unicode)
2184 d = node.get_child_at_path(name)
2186 self.failUnless(child.is_unknown() or not child.is_readonly())
2187 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2188 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2189 expected_ro_uri = self._make_readonly(got_uri)
2191 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2192 d.addCallback(_check)
2195 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2196 assert isinstance(name, unicode)
2197 d = node.get_child_at_path(name)
2199 self.failUnless(child.is_unknown() or child.is_readonly())
2200 self.failUnlessReallyEqual(child.get_write_uri(), None)
2201 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2202 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2203 d.addCallback(_check)
2206 def failUnlessCHKURIHasContents(self, got_uri, contents):
2207 self.failUnless(self.get_all_contents()[got_uri] == contents)
2209 def test_POST_upload(self):
2210 d = self.POST(self.public_url + "/foo", t="upload",
2211 file=("new.txt", self.NEWFILE_CONTENTS))
2213 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2214 d.addCallback(lambda res:
2215 self.failUnlessChildContentsAre(fn, u"new.txt",
2216 self.NEWFILE_CONTENTS))
2219 def test_POST_upload_unicode(self):
2220 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2221 d = self.POST(self.public_url + "/foo", t="upload",
2222 file=(filename, self.NEWFILE_CONTENTS))
2224 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2225 d.addCallback(lambda res:
2226 self.failUnlessChildContentsAre(fn, filename,
2227 self.NEWFILE_CONTENTS))
2228 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2229 d.addCallback(lambda res: self.GET(target_url))
2230 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2231 self.NEWFILE_CONTENTS,
2235 def test_POST_upload_unicode_named(self):
2236 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2237 d = self.POST(self.public_url + "/foo", t="upload",
2239 file=("overridden", self.NEWFILE_CONTENTS))
2241 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2242 d.addCallback(lambda res:
2243 self.failUnlessChildContentsAre(fn, filename,
2244 self.NEWFILE_CONTENTS))
2245 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2246 d.addCallback(lambda res: self.GET(target_url))
2247 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2248 self.NEWFILE_CONTENTS,
2252 def test_POST_upload_no_link(self):
2253 d = self.POST("/uri", t="upload",
2254 file=("new.txt", self.NEWFILE_CONTENTS))
2255 def _check_upload_results(page):
2256 # this should be a page which describes the results of the upload
2257 # that just finished.
2258 self.failUnlessIn("Upload Results:", page)
2259 self.failUnlessIn("URI:", page)
2260 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2261 mo = uri_re.search(page)
2262 self.failUnless(mo, page)
2263 new_uri = mo.group(1)
2265 d.addCallback(_check_upload_results)
2266 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2269 def test_POST_upload_no_link_whendone(self):
2270 d = self.POST("/uri", t="upload", when_done="/",
2271 file=("new.txt", self.NEWFILE_CONTENTS))
2272 d.addBoth(self.shouldRedirect, "/")
2275 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2276 d = defer.maybeDeferred(callable, *args, **kwargs)
2278 if isinstance(res, failure.Failure):
2279 res.trap(error.PageRedirect)
2280 statuscode = res.value.status
2281 target = res.value.location
2282 return checker(statuscode, target)
2283 self.fail("%s: callable was supposed to redirect, not return '%s'"
2288 def test_POST_upload_no_link_whendone_results(self):
2289 def check(statuscode, target):
2290 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2291 self.failUnless(target.startswith(self.webish_url), target)
2292 return client.getPage(target, method="GET")
2293 # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2294 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2296 self.POST, "/uri", t="upload",
2297 when_done="/%75ri/%(uri)s",
2298 file=("new.txt", self.NEWFILE_CONTENTS))
2299 d.addCallback(lambda res:
2300 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2303 def test_POST_upload_no_link_mutable(self):
2304 d = self.POST("/uri", t="upload", mutable="true",
2305 file=("new.txt", self.NEWFILE_CONTENTS))
2306 def _check(filecap):
2307 filecap = filecap.strip()
2308 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2309 self.filecap = filecap
2310 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2311 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2312 n = self.s.create_node_from_uri(filecap)
2313 return n.download_best_version()
2314 d.addCallback(_check)
2316 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2317 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2318 d.addCallback(_check2)
2320 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2321 return self.GET("/file/%s" % urllib.quote(self.filecap))
2322 d.addCallback(_check3)
2324 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2325 d.addCallback(_check4)
2328 def test_POST_upload_no_link_mutable_toobig(self):
2329 # The SDMF size limit is no longer in place, so we should be
2330 # able to upload mutable files that are as large as we want them
2332 d = self.POST("/uri", t="upload", mutable="true",
2333 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2337 def test_POST_upload_format_unlinked(self):
2338 def _check_upload_unlinked(ign, format, uri_prefix):
2339 filename = format + ".txt"
2340 d = self.POST("/uri?t=upload&format=" + format,
2341 file=(filename, self.NEWFILE_CONTENTS * 300000))
2342 def _got_results(results):
2343 if format.upper() in ("SDMF", "MDMF"):
2344 # webapi.rst says this returns a filecap
2347 # for immutable, it returns an "upload results page", and
2348 # the filecap is buried inside
2349 line = [l for l in results.split("\n") if "URI: " in l][0]
2350 mo = re.search(r'<span>([^<]+)</span>', line)
2351 filecap = mo.group(1)
2352 self.failUnless(filecap.startswith(uri_prefix),
2353 (uri_prefix, filecap))
2354 return self.GET("/uri/%s?t=json" % filecap)
2355 d.addCallback(_got_results)
2356 def _got_json(json):
2357 data = simplejson.loads(json)
2359 self.failUnlessIn("format", data)
2360 self.failUnlessEqual(data["format"], format.upper())
2361 d.addCallback(_got_json)
2363 d = defer.succeed(None)
2364 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2365 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2366 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2367 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2370 def test_POST_upload_bad_format_unlinked(self):
2371 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2372 400, "Bad Request", "Unknown format: foo",
2374 "/uri?t=upload&format=foo",
2375 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2377 def test_POST_upload_format(self):
2378 def _check_upload(ign, format, uri_prefix, fn=None):
2379 filename = format + ".txt"
2380 d = self.POST(self.public_url +
2381 "/foo?t=upload&format=" + format,
2382 file=(filename, self.NEWFILE_CONTENTS * 300000))
2383 def _got_filecap(filecap):
2385 filenameu = unicode(filename)
2386 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2387 self.failUnless(filecap.startswith(uri_prefix))
2388 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2389 d.addCallback(_got_filecap)
2390 def _got_json(json):
2391 data = simplejson.loads(json)
2393 self.failUnlessIn("format", data)
2394 self.failUnlessEqual(data["format"], format.upper())
2395 d.addCallback(_got_json)
2398 d = defer.succeed(None)
2399 d.addCallback(_check_upload, "chk", "URI:CHK")
2400 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2401 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2402 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2405 def test_POST_upload_bad_format(self):
2406 return self.shouldHTTPError("POST_upload_bad_format",
2407 400, "Bad Request", "Unknown format: foo",
2408 self.POST, self.public_url + \
2409 "/foo?t=upload&format=foo",
2410 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2412 def test_POST_upload_mutable(self):
2413 # this creates a mutable file
2414 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2415 file=("new.txt", self.NEWFILE_CONTENTS))
2417 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2418 d.addCallback(lambda res:
2419 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2420 self.NEWFILE_CONTENTS))
2421 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2423 self.failUnless(IMutableFileNode.providedBy(newnode))
2424 self.failUnless(newnode.is_mutable())
2425 self.failIf(newnode.is_readonly())
2426 self._mutable_node = newnode
2427 self._mutable_uri = newnode.get_uri()
2430 # now upload it again and make sure that the URI doesn't change
2431 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2432 d.addCallback(lambda res:
2433 self.POST(self.public_url + "/foo", t="upload",
2435 file=("new.txt", NEWER_CONTENTS)))
2436 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2437 d.addCallback(lambda res:
2438 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2440 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2442 self.failUnless(IMutableFileNode.providedBy(newnode))
2443 self.failUnless(newnode.is_mutable())
2444 self.failIf(newnode.is_readonly())
2445 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2446 d.addCallback(_got2)
2448 # upload a second time, using PUT instead of POST
2449 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2450 d.addCallback(lambda res:
2451 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2452 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2453 d.addCallback(lambda res:
2454 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2457 # finally list the directory, since mutable files are displayed
2458 # slightly differently
2460 d.addCallback(lambda res:
2461 self.GET(self.public_url + "/foo/",
2462 followRedirect=True))
2463 def _check_page(res):
2464 # TODO: assert more about the contents
2465 self.failUnlessIn("SSK", res)
2467 d.addCallback(_check_page)
2469 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2471 self.failUnless(IMutableFileNode.providedBy(newnode))
2472 self.failUnless(newnode.is_mutable())
2473 self.failIf(newnode.is_readonly())
2474 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2475 d.addCallback(_got3)
2477 # look at the JSON form of the enclosing directory
2478 d.addCallback(lambda res:
2479 self.GET(self.public_url + "/foo/?t=json",
2480 followRedirect=True))
2481 def _check_page_json(res):
2482 parsed = simplejson.loads(res)
2483 self.failUnlessEqual(parsed[0], "dirnode")
2484 children = dict( [(unicode(name),value)
2486 in parsed[1]["children"].iteritems()] )
2487 self.failUnlessIn(u"new.txt", children)
2488 new_json = children[u"new.txt"]
2489 self.failUnlessEqual(new_json[0], "filenode")
2490 self.failUnless(new_json[1]["mutable"])
2491 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2492 ro_uri = self._mutable_node.get_readonly().to_string()
2493 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2494 d.addCallback(_check_page_json)
2496 # and the JSON form of the file
2497 d.addCallback(lambda res:
2498 self.GET(self.public_url + "/foo/new.txt?t=json"))
2499 def _check_file_json(res):
2500 parsed = simplejson.loads(res)
2501 self.failUnlessEqual(parsed[0], "filenode")
2502 self.failUnless(parsed[1]["mutable"])
2503 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2504 ro_uri = self._mutable_node.get_readonly().to_string()
2505 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2506 d.addCallback(_check_file_json)
2508 # and look at t=uri and t=readonly-uri
2509 d.addCallback(lambda res:
2510 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2511 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2512 d.addCallback(lambda res:
2513 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2514 def _check_ro_uri(res):
2515 ro_uri = self._mutable_node.get_readonly().to_string()
2516 self.failUnlessReallyEqual(res, ro_uri)
2517 d.addCallback(_check_ro_uri)
2519 # make sure we can get to it from /uri/URI
2520 d.addCallback(lambda res:
2521 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2522 d.addCallback(lambda res:
2523 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2525 # and that HEAD computes the size correctly
2526 d.addCallback(lambda res:
2527 self.HEAD(self.public_url + "/foo/new.txt",
2528 return_response=True))
2529 def _got_headers((res, status, headers)):
2530 self.failUnlessReallyEqual(res, "")
2531 self.failUnlessReallyEqual(headers["content-length"][0],
2532 str(len(NEW2_CONTENTS)))
2533 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2534 d.addCallback(_got_headers)
2536 # make sure that outdated size limits aren't enforced anymore.
2537 d.addCallback(lambda ignored:
2538 self.POST(self.public_url + "/foo", t="upload",
2541 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2542 d.addErrback(self.dump_error)
2545 def test_POST_upload_mutable_toobig(self):
2546 # SDMF had a size limti that was removed a while ago. MDMF has
2547 # never had a size limit. Test to make sure that we do not
2548 # encounter errors when trying to upload large mutable files,
2549 # since there should be no coded prohibitions regarding large
2551 d = self.POST(self.public_url + "/foo",
2552 t="upload", mutable="true",
2553 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2556 def dump_error(self, f):
2557 # if the web server returns an error code (like 400 Bad Request),
2558 # web.client.getPage puts the HTTP response body into the .response
2559 # attribute of the exception object that it gives back. It does not
2560 # appear in the Failure's repr(), so the ERROR that trial displays
2561 # will be rather terse and unhelpful. addErrback this method to the
2562 # end of your chain to get more information out of these errors.
2563 if f.check(error.Error):
2564 print "web.error.Error:"
2566 print f.value.response
2569 def test_POST_upload_replace(self):
2570 d = self.POST(self.public_url + "/foo", t="upload",
2571 file=("bar.txt", self.NEWFILE_CONTENTS))
2573 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2574 d.addCallback(lambda res:
2575 self.failUnlessChildContentsAre(fn, u"bar.txt",
2576 self.NEWFILE_CONTENTS))
2579 def test_POST_upload_no_replace_ok(self):
2580 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2581 file=("new.txt", self.NEWFILE_CONTENTS))
2582 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2583 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2584 self.NEWFILE_CONTENTS))
2587 def test_POST_upload_no_replace_queryarg(self):
2588 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2589 file=("bar.txt", self.NEWFILE_CONTENTS))
2590 d.addBoth(self.shouldFail, error.Error,
2591 "POST_upload_no_replace_queryarg",
2593 "There was already a child by that name, and you asked me "
2594 "to not replace it")
2595 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2596 d.addCallback(self.failUnlessIsBarDotTxt)
2599 def test_POST_upload_no_replace_field(self):
2600 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2601 file=("bar.txt", self.NEWFILE_CONTENTS))
2602 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2604 "There was already a child by that name, and you asked me "
2605 "to not replace it")
2606 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2607 d.addCallback(self.failUnlessIsBarDotTxt)
2610 def test_POST_upload_whendone(self):
2611 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2612 file=("new.txt", self.NEWFILE_CONTENTS))
2613 d.addBoth(self.shouldRedirect, "/THERE")
2615 d.addCallback(lambda res:
2616 self.failUnlessChildContentsAre(fn, u"new.txt",
2617 self.NEWFILE_CONTENTS))
2620 def test_POST_upload_named(self):
2622 d = self.POST(self.public_url + "/foo", t="upload",
2623 name="new.txt", file=self.NEWFILE_CONTENTS)
2624 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2625 d.addCallback(lambda res:
2626 self.failUnlessChildContentsAre(fn, u"new.txt",
2627 self.NEWFILE_CONTENTS))
2630 def test_POST_upload_named_badfilename(self):
2631 d = self.POST(self.public_url + "/foo", t="upload",
2632 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2633 d.addBoth(self.shouldFail, error.Error,
2634 "test_POST_upload_named_badfilename",
2636 "name= may not contain a slash",
2638 # make sure that nothing was added
2639 d.addCallback(lambda res:
2640 self.failUnlessNodeKeysAre(self._foo_node,
2641 [self._htmlname_unicode,
2642 u"bar.txt", u"baz.txt", u"blockingfile",
2643 u"empty", u"n\u00fc.txt", u"quux.txt",
2647 def test_POST_FILEURL_check(self):
2648 bar_url = self.public_url + "/foo/bar.txt"
2649 d = self.POST(bar_url, t="check")
2651 self.failUnlessIn("Healthy :", res)
2652 d.addCallback(_check)
2653 redir_url = "http://allmydata.org/TARGET"
2654 def _check2(statuscode, target):
2655 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2656 self.failUnlessReallyEqual(target, redir_url)
2657 d.addCallback(lambda res:
2658 self.shouldRedirect2("test_POST_FILEURL_check",
2662 when_done=redir_url))
2663 d.addCallback(lambda res:
2664 self.POST(bar_url, t="check", return_to=redir_url))
2666 self.failUnlessIn("Healthy :", res)
2667 self.failUnlessIn("Return to file", res)
2668 self.failUnlessIn(redir_url, res)
2669 d.addCallback(_check3)
2671 d.addCallback(lambda res:
2672 self.POST(bar_url, t="check", output="JSON"))
2673 def _check_json(res):
2674 data = simplejson.loads(res)
2675 self.failUnlessIn("storage-index", data)
2676 self.failUnless(data["results"]["healthy"])
2677 d.addCallback(_check_json)
2681 def test_POST_FILEURL_check_and_repair(self):
2682 bar_url = self.public_url + "/foo/bar.txt"
2683 d = self.POST(bar_url, t="check", repair="true")
2685 self.failUnlessIn("Healthy :", res)
2686 d.addCallback(_check)
2687 redir_url = "http://allmydata.org/TARGET"
2688 def _check2(statuscode, target):
2689 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2690 self.failUnlessReallyEqual(target, redir_url)
2691 d.addCallback(lambda res:
2692 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2695 t="check", repair="true",
2696 when_done=redir_url))
2697 d.addCallback(lambda res:
2698 self.POST(bar_url, t="check", return_to=redir_url))
2700 self.failUnlessIn("Healthy :", res)
2701 self.failUnlessIn("Return to file", res)
2702 self.failUnlessIn(redir_url, res)
2703 d.addCallback(_check3)
2706 def test_POST_DIRURL_check(self):
2707 foo_url = self.public_url + "/foo/"
2708 d = self.POST(foo_url, t="check")
2710 self.failUnlessIn("Healthy :", res)
2711 d.addCallback(_check)
2712 redir_url = "http://allmydata.org/TARGET"
2713 def _check2(statuscode, target):
2714 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2715 self.failUnlessReallyEqual(target, redir_url)
2716 d.addCallback(lambda res:
2717 self.shouldRedirect2("test_POST_DIRURL_check",
2721 when_done=redir_url))
2722 d.addCallback(lambda res:
2723 self.POST(foo_url, t="check", return_to=redir_url))
2725 self.failUnlessIn("Healthy :", res)
2726 self.failUnlessIn("Return to file/directory", res)
2727 self.failUnlessIn(redir_url, res)
2728 d.addCallback(_check3)
2730 d.addCallback(lambda res:
2731 self.POST(foo_url, t="check", output="JSON"))
2732 def _check_json(res):
2733 data = simplejson.loads(res)
2734 self.failUnlessIn("storage-index", data)
2735 self.failUnless(data["results"]["healthy"])
2736 d.addCallback(_check_json)
2740 def test_POST_DIRURL_check_and_repair(self):
2741 foo_url = self.public_url + "/foo/"
2742 d = self.POST(foo_url, t="check", repair="true")
2744 self.failUnlessIn("Healthy :", res)
2745 d.addCallback(_check)
2746 redir_url = "http://allmydata.org/TARGET"
2747 def _check2(statuscode, target):
2748 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2749 self.failUnlessReallyEqual(target, redir_url)
2750 d.addCallback(lambda res:
2751 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2754 t="check", repair="true",
2755 when_done=redir_url))
2756 d.addCallback(lambda res:
2757 self.POST(foo_url, t="check", return_to=redir_url))
2759 self.failUnlessIn("Healthy :", res)
2760 self.failUnlessIn("Return to file/directory", res)
2761 self.failUnlessIn(redir_url, res)
2762 d.addCallback(_check3)
2765 def test_POST_FILEURL_mdmf_check(self):
2766 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2767 d = self.POST(quux_url, t="check")
2769 self.failUnlessIn("Healthy", res)
2770 d.addCallback(_check)
2771 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2772 d.addCallback(lambda ignored:
2773 self.POST(quux_extension_url, t="check"))
2774 d.addCallback(_check)
2777 def test_POST_FILEURL_mdmf_check_and_repair(self):
2778 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2779 d = self.POST(quux_url, t="check", repair="true")
2781 self.failUnlessIn("Healthy", res)
2782 d.addCallback(_check)
2783 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2784 d.addCallback(lambda ignored:
2785 self.POST(quux_extension_url, t="check", repair="true"))
2786 d.addCallback(_check)
2789 def wait_for_operation(self, ignored, ophandle):
2790 url = "/operations/" + ophandle
2791 url += "?t=status&output=JSON"
2794 data = simplejson.loads(res)
2795 if not data["finished"]:
2796 d = self.stall(delay=1.0)
2797 d.addCallback(self.wait_for_operation, ophandle)
2803 def get_operation_results(self, ignored, ophandle, output=None):
2804 url = "/operations/" + ophandle
2807 url += "&output=" + output
2810 if output and output.lower() == "json":
2811 return simplejson.loads(res)
2816 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2817 d = self.shouldFail2(error.Error,
2818 "test_POST_DIRURL_deepcheck_no_ophandle",
2820 "slow operation requires ophandle=",
2821 self.POST, self.public_url, t="start-deep-check")
2824 def test_POST_DIRURL_deepcheck(self):
2825 def _check_redirect(statuscode, target):
2826 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2827 self.failUnless(target.endswith("/operations/123"))
2828 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2829 self.POST, self.public_url,
2830 t="start-deep-check", ophandle="123")
2831 d.addCallback(self.wait_for_operation, "123")
2832 def _check_json(data):
2833 self.failUnlessReallyEqual(data["finished"], True)
2834 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2835 self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2836 d.addCallback(_check_json)
2837 d.addCallback(self.get_operation_results, "123", "html")
2838 def _check_html(res):
2839 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2840 self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2841 self.failUnlessIn(FAVICON_MARKUP, res)
2842 d.addCallback(_check_html)
2844 d.addCallback(lambda res:
2845 self.GET("/operations/123/"))
2846 d.addCallback(_check_html) # should be the same as without the slash
2848 d.addCallback(lambda res:
2849 self.shouldFail2(error.Error, "one", "404 Not Found",
2850 "No detailed results for SI bogus",
2851 self.GET, "/operations/123/bogus"))
2853 foo_si = self._foo_node.get_storage_index()
2854 foo_si_s = base32.b2a(foo_si)
2855 d.addCallback(lambda res:
2856 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2857 def _check_foo_json(res):
2858 data = simplejson.loads(res)
2859 self.failUnlessEqual(data["storage-index"], foo_si_s)
2860 self.failUnless(data["results"]["healthy"])
2861 d.addCallback(_check_foo_json)
2864 def test_POST_DIRURL_deepcheck_and_repair(self):
2865 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2866 ophandle="124", output="json", followRedirect=True)
2867 d.addCallback(self.wait_for_operation, "124")
2868 def _check_json(data):
2869 self.failUnlessReallyEqual(data["finished"], True)
2870 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2871 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2872 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2873 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2874 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2875 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2876 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2877 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2878 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2879 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2880 d.addCallback(_check_json)
2881 d.addCallback(self.get_operation_results, "124", "html")
2882 def _check_html(res):
2883 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2885 self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2886 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2887 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2889 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2890 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2891 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2893 self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2894 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2895 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2897 self.failUnlessIn(FAVICON_MARKUP, res)
2898 d.addCallback(_check_html)
2901 def test_POST_FILEURL_bad_t(self):
2902 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2903 "POST to file: bad t=bogus",
2904 self.POST, self.public_url + "/foo/bar.txt",
2908 def test_POST_mkdir(self): # return value?
2909 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2910 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2911 d.addCallback(self.failUnlessNodeKeysAre, [])
2914 def test_POST_mkdir_mdmf(self):
2915 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2916 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917 d.addCallback(lambda node:
2918 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2921 def test_POST_mkdir_sdmf(self):
2922 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2923 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2924 d.addCallback(lambda node:
2925 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2928 def test_POST_mkdir_bad_format(self):
2929 return self.shouldHTTPError("POST_mkdir_bad_format",
2930 400, "Bad Request", "Unknown format: foo",
2931 self.POST, self.public_url +
2932 "/foo?t=mkdir&name=newdir&format=foo")
2934 def test_POST_mkdir_initial_children(self):
2935 (newkids, caps) = self._create_initial_children()
2936 d = self.POST2(self.public_url +
2937 "/foo?t=mkdir-with-children&name=newdir",
2938 simplejson.dumps(newkids))
2939 d.addCallback(lambda res:
2940 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2941 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2942 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2943 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2944 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2947 def test_POST_mkdir_initial_children_mdmf(self):
2948 (newkids, caps) = self._create_initial_children()
2949 d = self.POST2(self.public_url +
2950 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2951 simplejson.dumps(newkids))
2952 d.addCallback(lambda res:
2953 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2954 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2955 d.addCallback(lambda node:
2956 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2957 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2958 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2963 def test_POST_mkdir_initial_children_sdmf(self):
2964 (newkids, caps) = self._create_initial_children()
2965 d = self.POST2(self.public_url +
2966 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2967 simplejson.dumps(newkids))
2968 d.addCallback(lambda res:
2969 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2970 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2971 d.addCallback(lambda node:
2972 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2973 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2974 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2978 def test_POST_mkdir_initial_children_bad_format(self):
2979 (newkids, caps) = self._create_initial_children()
2980 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2981 400, "Bad Request", "Unknown format: foo",
2982 self.POST, self.public_url + \
2983 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2984 simplejson.dumps(newkids))
2986 def test_POST_mkdir_immutable(self):
2987 (newkids, caps) = self._create_immutable_children()
2988 d = self.POST2(self.public_url +
2989 "/foo?t=mkdir-immutable&name=newdir",
2990 simplejson.dumps(newkids))
2991 d.addCallback(lambda res:
2992 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2993 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2994 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2995 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2996 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2997 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3007 def test_POST_mkdir_immutable_bad(self):
3008 (newkids, caps) = self._create_initial_children()
3009 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3011 "needed to be immutable but was not",
3014 "/foo?t=mkdir-immutable&name=newdir",
3015 simplejson.dumps(newkids))
3018 def test_POST_mkdir_2(self):
3019 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3020 d.addCallback(lambda res:
3021 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3022 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3023 d.addCallback(self.failUnlessNodeKeysAre, [])
3026 def test_POST_mkdirs_2(self):
3027 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3028 d.addCallback(lambda res:
3029 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3030 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3031 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3032 d.addCallback(self.failUnlessNodeKeysAre, [])
3035 def test_POST_mkdir_no_parentdir_noredirect(self):
3036 d = self.POST("/uri?t=mkdir")
3037 def _after_mkdir(res):
3038 uri.DirectoryURI.init_from_string(res)
3039 d.addCallback(_after_mkdir)
3042 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3043 d = self.POST("/uri?t=mkdir&format=mdmf")
3044 def _after_mkdir(res):
3045 u = uri.from_string(res)
3046 # Check that this is an MDMF writecap
3047 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3048 d.addCallback(_after_mkdir)
3051 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3052 d = self.POST("/uri?t=mkdir&format=sdmf")
3053 def _after_mkdir(res):
3054 u = uri.from_string(res)
3055 self.failUnlessIsInstance(u, uri.DirectoryURI)
3056 d.addCallback(_after_mkdir)
3059 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3060 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3061 400, "Bad Request", "Unknown format: foo",
3062 self.POST, self.public_url +
3063 "/uri?t=mkdir&format=foo")
3065 def test_POST_mkdir_no_parentdir_noredirect2(self):
3066 # make sure form-based arguments (as on the welcome page) still work
3067 d = self.POST("/uri", t="mkdir")
3068 def _after_mkdir(res):
3069 uri.DirectoryURI.init_from_string(res)
3070 d.addCallback(_after_mkdir)
3071 d.addErrback(self.explain_web_error)
3074 def test_POST_mkdir_no_parentdir_redirect(self):
3075 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3076 d.addBoth(self.shouldRedirect, None, statuscode='303')
3077 def _check_target(target):
3078 target = urllib.unquote(target)
3079 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3080 d.addCallback(_check_target)
3083 def test_POST_mkdir_no_parentdir_redirect2(self):
3084 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3085 d.addBoth(self.shouldRedirect, None, statuscode='303')
3086 def _check_target(target):
3087 target = urllib.unquote(target)
3088 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3089 d.addCallback(_check_target)
3090 d.addErrback(self.explain_web_error)
3093 def _make_readonly(self, u):
3094 ro_uri = uri.from_string(u).get_readonly()
3097 return ro_uri.to_string()
3099 def _create_initial_children(self):
3100 contents, n, filecap1 = self.makefile(12)
3101 md1 = {"metakey1": "metavalue1"}
3102 filecap2 = make_mutable_file_uri()
3103 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3104 filecap3 = node3.get_readonly_uri()
3105 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3106 dircap = DirectoryNode(node4, None, None).get_uri()
3107 mdmfcap = make_mutable_file_uri(mdmf=True)
3108 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3109 emptydircap = "URI:DIR2-LIT:"
3110 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3111 "ro_uri": self._make_readonly(filecap1),
3112 "metadata": md1, }],
3113 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3114 "ro_uri": self._make_readonly(filecap2)}],
3115 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3116 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3117 "ro_uri": unknown_rocap}],
3118 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3119 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3120 u"dirchild": ["dirnode", {"rw_uri": dircap,
3121 "ro_uri": self._make_readonly(dircap)}],
3122 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3123 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3124 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3125 "ro_uri": self._make_readonly(mdmfcap)}],
3127 return newkids, {'filecap1': filecap1,
3128 'filecap2': filecap2,
3129 'filecap3': filecap3,
3130 'unknown_rwcap': unknown_rwcap,
3131 'unknown_rocap': unknown_rocap,
3132 'unknown_immcap': unknown_immcap,
3134 'litdircap': litdircap,
3135 'emptydircap': emptydircap,
3138 def _create_immutable_children(self):
3139 contents, n, filecap1 = self.makefile(12)
3140 md1 = {"metakey1": "metavalue1"}
3141 tnode = create_chk_filenode("immutable directory contents\n"*10,
3142 self.get_all_contents())
3143 dnode = DirectoryNode(tnode, None, None)
3144 assert not dnode.is_mutable()
3145 immdircap = dnode.get_uri()
3146 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3147 emptydircap = "URI:DIR2-LIT:"
3148 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3149 "metadata": md1, }],
3150 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3151 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3152 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3153 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3155 return newkids, {'filecap1': filecap1,
3156 'unknown_immcap': unknown_immcap,
3157 'immdircap': immdircap,
3158 'litdircap': litdircap,
3159 'emptydircap': emptydircap}
3161 def test_POST_mkdir_no_parentdir_initial_children(self):
3162 (newkids, caps) = self._create_initial_children()
3163 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3164 def _after_mkdir(res):
3165 self.failUnless(res.startswith("URI:DIR"), res)
3166 n = self.s.create_node_from_uri(res)
3167 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3168 d2.addCallback(lambda ign:
3169 self.failUnlessROChildURIIs(n, u"child-imm",
3171 d2.addCallback(lambda ign:
3172 self.failUnlessRWChildURIIs(n, u"child-mutable",
3174 d2.addCallback(lambda ign:
3175 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3177 d2.addCallback(lambda ign:
3178 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3179 caps['unknown_rwcap']))
3180 d2.addCallback(lambda ign:
3181 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3182 caps['unknown_rocap']))
3183 d2.addCallback(lambda ign:
3184 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3185 caps['unknown_immcap']))
3186 d2.addCallback(lambda ign:
3187 self.failUnlessRWChildURIIs(n, u"dirchild",
3190 d.addCallback(_after_mkdir)
3193 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3194 # the regular /uri?t=mkdir operation is specified to ignore its body.
3195 # Only t=mkdir-with-children pays attention to it.
3196 (newkids, caps) = self._create_initial_children()
3197 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3199 "t=mkdir does not accept children=, "
3200 "try t=mkdir-with-children instead",
3201 self.POST2, "/uri?t=mkdir", # without children
3202 simplejson.dumps(newkids))
3205 def test_POST_noparent_bad(self):
3206 d = self.shouldHTTPError("POST_noparent_bad",
3208 "/uri accepts only PUT, PUT?t=mkdir, "
3209 "POST?t=upload, and POST?t=mkdir",
3210 self.POST, "/uri?t=bogus")
3213 def test_POST_mkdir_no_parentdir_immutable(self):
3214 (newkids, caps) = self._create_immutable_children()
3215 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3216 def _after_mkdir(res):
3217 self.failUnless(res.startswith("URI:DIR"), res)
3218 n = self.s.create_node_from_uri(res)
3219 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3220 d2.addCallback(lambda ign:
3221 self.failUnlessROChildURIIs(n, u"child-imm",
3223 d2.addCallback(lambda ign:
3224 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3225 caps['unknown_immcap']))
3226 d2.addCallback(lambda ign:
3227 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3229 d2.addCallback(lambda ign:
3230 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3232 d2.addCallback(lambda ign:
3233 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3234 caps['emptydircap']))
3236 d.addCallback(_after_mkdir)
3239 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3240 (newkids, caps) = self._create_initial_children()
3241 d = self.shouldFail2(error.Error,
3242 "test_POST_mkdir_no_parentdir_immutable_bad",
3244 "needed to be immutable but was not",
3246 "/uri?t=mkdir-immutable",
3247 simplejson.dumps(newkids))
3250 def test_welcome_page_mkdir_button(self):
3251 # Fetch the welcome page.
3253 def _after_get_welcome_page(res):
3254 MKDIR_BUTTON_RE = re.compile(
3255 '<form action="([^"]*)" method="post".*'
3256 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3257 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3258 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3259 html = res.replace('\n', ' ')
3260 mo = MKDIR_BUTTON_RE.search(html)
3261 self.failUnless(mo, html)
3262 formaction = mo.group(1)
3264 formaname = mo.group(3)
3265 formavalue = mo.group(4)
3266 return (formaction, formt, formaname, formavalue)
3267 d.addCallback(_after_get_welcome_page)
3268 def _after_parse_form(res):
3269 (formaction, formt, formaname, formavalue) = res
3270 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3271 d.addCallback(_after_parse_form)
3272 d.addBoth(self.shouldRedirect, None, statuscode='303')
3275 def test_POST_mkdir_replace(self): # return value?
3276 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3277 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3278 d.addCallback(self.failUnlessNodeKeysAre, [])
3281 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3282 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3283 d.addBoth(self.shouldFail, error.Error,
3284 "POST_mkdir_no_replace_queryarg",
3286 "There was already a child by that name, and you asked me "
3287 "to not replace it")
3288 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3289 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3292 def test_POST_mkdir_no_replace_field(self): # return value?
3293 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3295 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3297 "There was already a child by that name, and you asked me "
3298 "to not replace it")
3299 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3300 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3303 def test_POST_mkdir_whendone_field(self):
3304 d = self.POST(self.public_url + "/foo",
3305 t="mkdir", name="newdir", when_done="/THERE")
3306 d.addBoth(self.shouldRedirect, "/THERE")
3307 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3308 d.addCallback(self.failUnlessNodeKeysAre, [])
3311 def test_POST_mkdir_whendone_queryarg(self):
3312 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3313 t="mkdir", name="newdir")
3314 d.addBoth(self.shouldRedirect, "/THERE")
3315 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3316 d.addCallback(self.failUnlessNodeKeysAre, [])
3319 def test_POST_bad_t(self):
3320 d = self.shouldFail2(error.Error, "POST_bad_t",
3322 "POST to a directory with bad t=BOGUS",
3323 self.POST, self.public_url + "/foo", t="BOGUS")
3326 def test_POST_set_children(self, command_name="set_children"):
3327 contents9, n9, newuri9 = self.makefile(9)
3328 contents10, n10, newuri10 = self.makefile(10)
3329 contents11, n11, newuri11 = self.makefile(11)
3332 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3335 "ctime": 1002777696.7564139,
3336 "mtime": 1002777696.7564139
3339 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3342 "ctime": 1002777696.7564139,
3343 "mtime": 1002777696.7564139
3346 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3349 "ctime": 1002777696.7564139,
3350 "mtime": 1002777696.7564139
3353 }""" % (newuri9, newuri10, newuri11)
3355 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3357 d = client.getPage(url, method="POST", postdata=reqbody)
3359 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3360 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3361 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3363 d.addCallback(_then)
3364 d.addErrback(self.dump_error)
3367 def test_POST_set_children_with_hyphen(self):
3368 return self.test_POST_set_children(command_name="set-children")
3370 def test_POST_link_uri(self):
3371 contents, n, newuri = self.makefile(8)
3372 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3373 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3374 d.addCallback(lambda res:
3375 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3379 def test_POST_link_uri_replace(self):
3380 contents, n, newuri = self.makefile(8)
3381 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3382 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3383 d.addCallback(lambda res:
3384 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3388 def test_POST_link_uri_unknown_bad(self):
3389 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3390 d.addBoth(self.shouldFail, error.Error,
3391 "POST_link_uri_unknown_bad",
3393 "unknown cap in a write slot")
3396 def test_POST_link_uri_unknown_ro_good(self):
3397 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3398 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3401 def test_POST_link_uri_unknown_imm_good(self):
3402 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3403 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3406 def test_POST_link_uri_no_replace_queryarg(self):
3407 contents, n, newuri = self.makefile(8)
3408 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3409 name="bar.txt", uri=newuri)
3410 d.addBoth(self.shouldFail, error.Error,
3411 "POST_link_uri_no_replace_queryarg",
3413 "There was already a child by that name, and you asked me "
3414 "to not replace it")
3415 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3416 d.addCallback(self.failUnlessIsBarDotTxt)
3419 def test_POST_link_uri_no_replace_field(self):
3420 contents, n, newuri = self.makefile(8)
3421 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3422 name="bar.txt", uri=newuri)
3423 d.addBoth(self.shouldFail, error.Error,
3424 "POST_link_uri_no_replace_field",
3426 "There was already a child by that name, and you asked me "
3427 "to not replace it")
3428 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3429 d.addCallback(self.failUnlessIsBarDotTxt)
3432 def test_POST_delete(self, command_name='delete'):
3433 d = self._foo_node.list()
3434 def _check_before(children):
3435 self.failUnlessIn(u"bar.txt", children)
3436 d.addCallback(_check_before)
3437 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3438 d.addCallback(lambda res: self._foo_node.list())
3439 def _check_after(children):
3440 self.failIfIn(u"bar.txt", children)
3441 d.addCallback(_check_after)
3444 def test_POST_unlink(self):
3445 return self.test_POST_delete(command_name='unlink')
3447 def test_POST_rename_file(self):
3448 d = self.POST(self.public_url + "/foo", t="rename",
3449 from_name="bar.txt", to_name='wibble.txt')
3450 d.addCallback(lambda res:
3451 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3452 d.addCallback(lambda res:
3453 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3454 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3455 d.addCallback(self.failUnlessIsBarDotTxt)
3456 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3457 d.addCallback(self.failUnlessIsBarJSON)
3460 def test_POST_rename_file_redundant(self):
3461 d = self.POST(self.public_url + "/foo", t="rename",
3462 from_name="bar.txt", to_name='bar.txt')
3463 d.addCallback(lambda res:
3464 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3465 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3466 d.addCallback(self.failUnlessIsBarDotTxt)
3467 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3468 d.addCallback(self.failUnlessIsBarJSON)
3471 def test_POST_rename_file_replace(self):
3472 # rename a file and replace a directory with it
3473 d = self.POST(self.public_url + "/foo", t="rename",
3474 from_name="bar.txt", to_name='empty')
3475 d.addCallback(lambda res:
3476 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3477 d.addCallback(lambda res:
3478 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3479 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3480 d.addCallback(self.failUnlessIsBarDotTxt)
3481 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3482 d.addCallback(self.failUnlessIsBarJSON)
3485 def test_POST_rename_file_no_replace_queryarg(self):
3486 # rename a file and replace a directory with it
3487 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3488 from_name="bar.txt", to_name='empty')
3489 d.addBoth(self.shouldFail, error.Error,
3490 "POST_rename_file_no_replace_queryarg",
3492 "There was already a child by that name, and you asked me "
3493 "to not replace it")
3494 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3495 d.addCallback(self.failUnlessIsEmptyJSON)
3498 def test_POST_rename_file_no_replace_field(self):
3499 # rename a file and replace a directory with it
3500 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3501 from_name="bar.txt", to_name='empty')
3502 d.addBoth(self.shouldFail, error.Error,
3503 "POST_rename_file_no_replace_field",
3505 "There was already a child by that name, and you asked me "
3506 "to not replace it")
3507 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3508 d.addCallback(self.failUnlessIsEmptyJSON)
3511 def test_POST_rename_file_no_replace_same_link(self):
3512 d = self.POST(self.public_url + "/foo", t="rename",
3513 replace="false", from_name="bar.txt", to_name="bar.txt")
3514 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3515 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3516 d.addCallback(self.failUnlessIsBarDotTxt)
3517 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3518 d.addCallback(self.failUnlessIsBarJSON)
3521 def test_POST_rename_file_replace_only_files(self):
3522 d = self.POST(self.public_url + "/foo", t="rename",
3523 replace="only-files", from_name="bar.txt",
3525 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3526 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3527 d.addCallback(self.failUnlessIsBarDotTxt)
3528 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3529 d.addCallback(self.failUnlessIsBarJSON)
3532 def test_POST_rename_file_replace_only_files_conflict(self):
3533 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3535 "There was already a child by that name, and you asked me to not replace it.",
3536 self.POST, self.public_url + "/foo", t="relink",
3537 replace="only-files", from_name="bar.txt",
3539 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3540 d.addCallback(self.failUnlessIsBarDotTxt)
3541 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3542 d.addCallback(self.failUnlessIsBarJSON)
3545 def failUnlessIsEmptyJSON(self, res):
3546 data = simplejson.loads(res)
3547 self.failUnlessEqual(data[0], "dirnode", data)
3548 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3550 def test_POST_rename_file_to_slash_fail(self):
3551 d = self.POST(self.public_url + "/foo", t="rename",
3552 from_name="bar.txt", to_name='kirk/spock.txt')
3553 d.addBoth(self.shouldFail, error.Error,
3554 "test_POST_rename_file_to_slash_fail",
3556 "to_name= may not contain a slash",
3558 d.addCallback(lambda res:
3559 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3562 def test_POST_rename_file_from_slash_fail(self):
3563 d = self.POST(self.public_url + "/foo", t="rename",
3564 from_name="sub/bar.txt", to_name='spock.txt')
3565 d.addBoth(self.shouldFail, error.Error,
3566 "test_POST_rename_from_file_slash_fail",
3568 "from_name= may not contain a slash",
3570 d.addCallback(lambda res:
3571 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3574 def test_POST_rename_dir(self):
3575 d = self.POST(self.public_url, t="rename",
3576 from_name="foo", to_name='plunk')
3577 d.addCallback(lambda res:
3578 self.failIfNodeHasChild(self.public_root, u"foo"))
3579 d.addCallback(lambda res:
3580 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3581 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3582 d.addCallback(self.failUnlessIsFooJSON)
3585 def test_POST_relink_file(self):
3586 d = self.POST(self.public_url + "/foo", t="relink",
3587 from_name="bar.txt",
3588 to_dir=self.public_root.get_uri() + "/foo/sub")
3589 d.addCallback(lambda res:
3590 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3591 d.addCallback(lambda res:
3592 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3593 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3594 d.addCallback(self.failUnlessIsBarDotTxt)
3595 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3596 d.addCallback(self.failUnlessIsBarJSON)
3599 def test_POST_relink_file_new_name(self):
3600 d = self.POST(self.public_url + "/foo", t="relink",
3601 from_name="bar.txt",
3602 to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3603 d.addCallback(lambda res:
3604 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3605 d.addCallback(lambda res:
3606 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3607 d.addCallback(lambda res:
3608 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3609 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3610 d.addCallback(self.failUnlessIsBarDotTxt)
3611 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3612 d.addCallback(self.failUnlessIsBarJSON)
3615 def test_POST_relink_file_replace(self):
3616 d = self.POST(self.public_url + "/foo", t="relink",
3617 from_name="bar.txt",
3618 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3619 d.addCallback(lambda res:
3620 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3621 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3622 d.addCallback(self.failUnlessIsBarDotTxt)
3623 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3624 d.addCallback(self.failUnlessIsBarJSON)
3627 def test_POST_relink_file_no_replace(self):
3628 d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3630 "There was already a child by that name, and you asked me to not replace it",
3631 self.POST, self.public_url + "/foo", t="relink",
3632 replace="false", from_name="bar.txt",
3633 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3634 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3635 d.addCallback(self.failUnlessIsBarDotTxt)
3636 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3637 d.addCallback(self.failUnlessIsBarJSON)
3638 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3639 d.addCallback(self.failUnlessIsSubBazDotTxt)
3642 def test_POST_relink_file_no_replace_explicitly_same_link(self):
3643 d = self.POST(self.public_url + "/foo", t="relink",
3644 replace="false", from_name="bar.txt",
3645 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3646 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3647 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3648 d.addCallback(self.failUnlessIsBarDotTxt)
3649 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3650 d.addCallback(self.failUnlessIsBarJSON)
3653 def test_POST_relink_file_replace_only_files(self):
3654 d = self.POST(self.public_url + "/foo", t="relink",
3655 replace="only-files", from_name="bar.txt",
3656 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3657 d.addCallback(lambda res:
3658 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3659 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3660 d.addCallback(self.failUnlessIsBarDotTxt)
3661 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3662 d.addCallback(self.failUnlessIsBarJSON)
3665 def test_POST_relink_file_replace_only_files_conflict(self):
3666 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3668 "There was already a child by that name, and you asked me to not replace it.",
3669 self.POST, self.public_url + "/foo", t="relink",
3670 replace="only-files", from_name="bar.txt",
3671 to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3672 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3673 d.addCallback(self.failUnlessIsBarDotTxt)
3674 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3675 d.addCallback(self.failUnlessIsBarJSON)
3678 def test_POST_relink_file_to_slash_fail(self):
3679 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3681 "to_name= may not contain a slash",
3682 self.POST, self.public_url + "/foo", t="relink",
3683 from_name="bar.txt",
3684 to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3685 d.addCallback(lambda res:
3686 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3687 d.addCallback(lambda res:
3688 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3689 d.addCallback(lambda ign:
3690 self.shouldFail2(error.Error,
3691 "test_POST_rename_file_slash_fail2",
3693 "from_name= may not contain a slash",
3694 self.POST, self.public_url + "/foo",
3696 from_name="nope/bar.txt",
3698 to_dir=self.public_root.get_uri() + "/foo/sub"))
3701 def test_POST_relink_file_explicitly_same_link(self):
3702 d = self.POST(self.public_url + "/foo", t="relink",
3703 from_name="bar.txt",
3704 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3705 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3706 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3707 d.addCallback(self.failUnlessIsBarDotTxt)
3708 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3709 d.addCallback(self.failUnlessIsBarJSON)
3712 def test_POST_relink_file_implicitly_same_link(self):
3713 d = self.POST(self.public_url + "/foo", t="relink",
3714 from_name="bar.txt")
3715 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3716 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3717 d.addCallback(self.failUnlessIsBarDotTxt)
3718 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3719 d.addCallback(self.failUnlessIsBarJSON)
3722 def test_POST_relink_file_same_dir(self):
3723 d = self.POST(self.public_url + "/foo", t="relink",
3724 from_name="bar.txt",
3725 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3726 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3727 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3728 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3729 d.addCallback(self.failUnlessIsBarDotTxt)
3730 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3731 d.addCallback(self.failUnlessIsBarJSON)
3734 def test_POST_relink_file_bad_replace(self):
3735 d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3736 "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3738 self.public_url + "/foo", t="relink",
3739 replace="boogabooga", from_name="bar.txt",
3740 to_dir=self.public_root.get_uri() + "/foo/sub")
3743 def test_POST_relink_file_multi_level(self):
3744 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3745 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3746 from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3747 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3748 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3749 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3750 d.addCallback(self.failUnlessIsBarDotTxt)
3751 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3752 d.addCallback(self.failUnlessIsBarJSON)
3755 def test_POST_relink_file_to_uri(self):
3756 d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3757 from_name="bar.txt", to_dir=self._sub_uri)
3758 d.addCallback(lambda res:
3759 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3760 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3761 d.addCallback(self.failUnlessIsBarDotTxt)
3762 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3763 d.addCallback(self.failUnlessIsBarJSON)
3766 def test_POST_relink_file_to_nonexistent_dir(self):
3767 d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3768 "404 Not Found", "No such child: nopechucktesta",
3769 self.POST, self.public_url + "/foo", t="relink",
3770 from_name="bar.txt",
3771 to_dir=self.public_root.get_uri() + "/nopechucktesta")
3774 def test_POST_relink_file_into_file(self):
3775 d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3776 "400 Bad Request", "to_dir is not a directory",
3777 self.POST, self.public_url + "/foo", t="relink",
3778 from_name="bar.txt",
3779 to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3780 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3781 d.addCallback(self.failUnlessIsBazDotTxt)
3782 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3783 d.addCallback(self.failUnlessIsBarDotTxt)
3784 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3785 d.addCallback(self.failUnlessIsBarJSON)
3788 def test_POST_relink_file_to_bad_uri(self):
3789 d = self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3790 "400 Bad Request", "to_dir is not a directory",
3791 self.POST, self.public_url + "/foo", t="relink",
3792 from_name="bar.txt",
3793 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3794 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3795 d.addCallback(self.failUnlessIsBarDotTxt)
3796 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3797 d.addCallback(self.failUnlessIsBarJSON)
3800 def test_POST_relink_dir(self):
3801 d = self.POST(self.public_url + "/foo", t="relink",
3802 from_name="bar.txt",
3803 to_dir=self.public_root.get_uri() + "/foo/empty")
3804 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3805 t="relink", from_name="empty",
3806 to_dir=self.public_root.get_uri() + "/foo/sub"))
3807 d.addCallback(lambda res:
3808 self.failIfNodeHasChild(self._foo_node, u"empty"))
3809 d.addCallback(lambda res:
3810 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3811 d.addCallback(lambda res:
3812 self._sub_node.get_child_at_path(u"empty"))
3813 d.addCallback(lambda node:
3814 self.failUnlessNodeHasChild(node, u"bar.txt"))
3815 d.addCallback(lambda res:
3816 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3817 d.addCallback(self.failUnlessIsBarDotTxt)
3820 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3821 """ If target is not None then the redirection has to go to target. If
3822 statuscode is not None then the redirection has to be accomplished with
3823 that HTTP status code."""
3824 if not isinstance(res, failure.Failure):
3825 to_where = (target is None) and "somewhere" or ("to " + target)
3826 self.fail("%s: we were expecting to get redirected %s, not get an"
3827 " actual page: %s" % (which, to_where, res))
3828 res.trap(error.PageRedirect)
3829 if statuscode is not None:
3830 self.failUnlessReallyEqual(res.value.status, statuscode,
3831 "%s: not a redirect" % which)
3832 if target is not None:
3833 # the PageRedirect does not seem to capture the uri= query arg
3834 # properly, so we can't check for it.
3835 realtarget = self.webish_url + target
3836 self.failUnlessReallyEqual(res.value.location, realtarget,
3837 "%s: wrong target" % which)
3838 return res.value.location
3840 def test_GET_URI_form(self):
3841 base = "/uri?uri=%s" % self._bar_txt_uri
3842 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3843 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3845 d.addBoth(self.shouldRedirect, targetbase)
3846 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3847 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3848 d.addCallback(lambda res: self.GET(base+"&t=json"))
3849 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3850 d.addCallback(self.log, "about to get file by uri")
3851 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3852 d.addCallback(self.failUnlessIsBarDotTxt)
3853 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3854 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3855 followRedirect=True))
3856 d.addCallback(self.failUnlessIsFooJSON)
3857 d.addCallback(self.log, "got dir by uri")
3861 def test_GET_URI_form_bad(self):
3862 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3863 "400 Bad Request", "GET /uri requires uri=",
3867 def test_GET_rename_form(self):
3868 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3869 followRedirect=True)
3871 self.failUnlessIn('name="when_done" value="."', res)
3872 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3873 self.failUnlessIn(FAVICON_MARKUP, res)
3874 d.addCallback(_check)
3877 def log(self, res, msg):
3878 #print "MSG: %s RES: %s" % (msg, res)
3882 def test_GET_URI_URL(self):
3883 base = "/uri/%s" % self._bar_txt_uri
3885 d.addCallback(self.failUnlessIsBarDotTxt)
3886 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3887 d.addCallback(self.failUnlessIsBarDotTxt)
3888 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3889 d.addCallback(self.failUnlessIsBarDotTxt)
3892 def test_GET_URI_URL_dir(self):
3893 base = "/uri/%s?t=json" % self._foo_uri
3895 d.addCallback(self.failUnlessIsFooJSON)
3898 def test_GET_URI_URL_missing(self):
3899 base = "/uri/%s" % self._bad_file_uri
3900 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3901 http.GONE, None, "NotEnoughSharesError",
3903 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3904 # here? we must arrange for a download to fail after target.open()
3905 # has been called, and then inspect the response to see that it is
3906 # shorter than we expected.
3909 def test_PUT_DIRURL_uri(self):
3910 d = self.s.create_dirnode()
3912 new_uri = dn.get_uri()
3913 # replace /foo with a new (empty) directory
3914 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3915 d.addCallback(lambda res:
3916 self.failUnlessReallyEqual(res.strip(), new_uri))
3917 d.addCallback(lambda res:
3918 self.failUnlessRWChildURIIs(self.public_root,
3922 d.addCallback(_made_dir)
3925 def test_PUT_DIRURL_uri_noreplace(self):
3926 d = self.s.create_dirnode()
3928 new_uri = dn.get_uri()
3929 # replace /foo with a new (empty) directory, but ask that
3930 # replace=false, so it should fail
3931 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3932 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3934 self.public_url + "/foo?t=uri&replace=false",
3936 d.addCallback(lambda res:
3937 self.failUnlessRWChildURIIs(self.public_root,
3941 d.addCallback(_made_dir)
3944 def test_PUT_DIRURL_bad_t(self):
3945 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3946 "400 Bad Request", "PUT to a directory",
3947 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3948 d.addCallback(lambda res:
3949 self.failUnlessRWChildURIIs(self.public_root,
3954 def test_PUT_NEWFILEURL_uri(self):
3955 contents, n, new_uri = self.makefile(8)
3956 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3957 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3958 d.addCallback(lambda res:
3959 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3963 def test_PUT_NEWFILEURL_mdmf(self):
3964 new_contents = self.NEWFILE_CONTENTS * 300000
3965 d = self.PUT(self.public_url + \
3966 "/foo/mdmf.txt?format=mdmf",
3968 d.addCallback(lambda ignored:
3969 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3970 def _got_json(json):
3971 data = simplejson.loads(json)
3973 self.failUnlessIn("format", data)
3974 self.failUnlessEqual(data["format"], "MDMF")
3975 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3976 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3977 d.addCallback(_got_json)
3980 def test_PUT_NEWFILEURL_sdmf(self):
3981 new_contents = self.NEWFILE_CONTENTS * 300000
3982 d = self.PUT(self.public_url + \
3983 "/foo/sdmf.txt?format=sdmf",
3985 d.addCallback(lambda ignored:
3986 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3987 def _got_json(json):
3988 data = simplejson.loads(json)
3990 self.failUnlessIn("format", data)
3991 self.failUnlessEqual(data["format"], "SDMF")
3992 d.addCallback(_got_json)
3995 def test_PUT_NEWFILEURL_bad_format(self):
3996 new_contents = self.NEWFILE_CONTENTS * 300000
3997 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3998 400, "Bad Request", "Unknown format: foo",
3999 self.PUT, self.public_url + \
4000 "/foo/foo.txt?format=foo",
4003 def test_PUT_NEWFILEURL_uri_replace(self):
4004 contents, n, new_uri = self.makefile(8)
4005 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4006 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4007 d.addCallback(lambda res:
4008 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4012 def test_PUT_NEWFILEURL_uri_no_replace(self):
4013 contents, n, new_uri = self.makefile(8)
4014 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4015 d.addBoth(self.shouldFail, error.Error,
4016 "PUT_NEWFILEURL_uri_no_replace",
4018 "There was already a child by that name, and you asked me "
4019 "to not replace it")
4022 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4023 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4024 d.addBoth(self.shouldFail, error.Error,
4025 "POST_put_uri_unknown_bad",
4027 "unknown cap in a write slot")
4030 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4031 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4032 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4033 u"put-future-ro.txt")
4036 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4037 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4038 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4039 u"put-future-imm.txt")
4042 def test_PUT_NEWFILE_URI(self):
4043 file_contents = "New file contents here\n"
4044 d = self.PUT("/uri", file_contents)
4046 assert isinstance(uri, str), uri
4047 self.failUnlessIn(uri, self.get_all_contents())
4048 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4050 return self.GET("/uri/%s" % uri)
4051 d.addCallback(_check)
4053 self.failUnlessReallyEqual(res, file_contents)
4054 d.addCallback(_check2)
4057 def test_PUT_NEWFILE_URI_not_mutable(self):
4058 file_contents = "New file contents here\n"
4059 d = self.PUT("/uri?mutable=false", file_contents)
4061 assert isinstance(uri, str), uri
4062 self.failUnlessIn(uri, self.get_all_contents())
4063 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4065 return self.GET("/uri/%s" % uri)
4066 d.addCallback(_check)
4068 self.failUnlessReallyEqual(res, file_contents)
4069 d.addCallback(_check2)
4072 def test_PUT_NEWFILE_URI_only_PUT(self):
4073 d = self.PUT("/uri?t=bogus", "")
4074 d.addBoth(self.shouldFail, error.Error,
4075 "PUT_NEWFILE_URI_only_PUT",
4077 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4080 def test_PUT_NEWFILE_URI_mutable(self):
4081 file_contents = "New file contents here\n"
4082 d = self.PUT("/uri?mutable=true", file_contents)
4083 def _check1(filecap):
4084 filecap = filecap.strip()
4085 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4086 self.filecap = filecap
4087 u = uri.WriteableSSKFileURI.init_from_string(filecap)
4088 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4089 n = self.s.create_node_from_uri(filecap)
4090 return n.download_best_version()
4091 d.addCallback(_check1)
4093 self.failUnlessReallyEqual(data, file_contents)
4094 return self.GET("/uri/%s" % urllib.quote(self.filecap))
4095 d.addCallback(_check2)
4097 self.failUnlessReallyEqual(res, file_contents)
4098 d.addCallback(_check3)
4101 def test_PUT_mkdir(self):
4102 d = self.PUT("/uri?t=mkdir", "")
4104 n = self.s.create_node_from_uri(uri.strip())
4105 d2 = self.failUnlessNodeKeysAre(n, [])
4106 d2.addCallback(lambda res:
4107 self.GET("/uri/%s?t=json" % uri))
4109 d.addCallback(_check)
4110 d.addCallback(self.failUnlessIsEmptyJSON)
4113 def test_PUT_mkdir_mdmf(self):
4114 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4116 u = uri.from_string(res)
4117 # Check that this is an MDMF writecap
4118 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4122 def test_PUT_mkdir_sdmf(self):
4123 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4125 u = uri.from_string(res)
4126 self.failUnlessIsInstance(u, uri.DirectoryURI)
4130 def test_PUT_mkdir_bad_format(self):
4131 return self.shouldHTTPError("PUT_mkdir_bad_format",
4132 400, "Bad Request", "Unknown format: foo",
4133 self.PUT, "/uri?t=mkdir&format=foo",
4136 def test_POST_check(self):
4137 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4139 # this returns a string form of the results, which are probably
4140 # None since we're using fake filenodes.
4141 # TODO: verify that the check actually happened, by changing
4142 # FakeCHKFileNode to count how many times .check() has been
4145 d.addCallback(_done)
4149 def test_PUT_update_at_offset(self):
4150 file_contents = "test file" * 100000 # about 900 KiB
4151 d = self.PUT("/uri?mutable=true", file_contents)
4153 self.filecap = filecap
4154 new_data = file_contents[:100]
4155 new = "replaced and so on"
4157 new_data += file_contents[len(new_data):]
4158 assert len(new_data) == len(file_contents)
4159 self.new_data = new_data
4160 d.addCallback(_then)
4161 d.addCallback(lambda ignored:
4162 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4163 "replaced and so on"))
4164 def _get_data(filecap):
4165 n = self.s.create_node_from_uri(filecap)
4166 return n.download_best_version()
4167 d.addCallback(_get_data)
4168 d.addCallback(lambda results:
4169 self.failUnlessEqual(results, self.new_data))
4170 # Now try appending things to the file
4171 d.addCallback(lambda ignored:
4172 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4174 d.addCallback(_get_data)
4175 d.addCallback(lambda results:
4176 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4177 # and try replacing the beginning of the file
4178 d.addCallback(lambda ignored:
4179 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4180 d.addCallback(_get_data)
4181 d.addCallback(lambda results:
4182 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4185 def test_PUT_update_at_invalid_offset(self):
4186 file_contents = "test file" * 100000 # about 900 KiB
4187 d = self.PUT("/uri?mutable=true", file_contents)
4189 self.filecap = filecap
4190 d.addCallback(_then)
4191 # Negative offsets should cause an error.
4192 d.addCallback(lambda ignored:
4193 self.shouldHTTPError("PUT_update_at_invalid_offset",
4197 "/uri/%s?offset=-1" % self.filecap,
4201 def test_PUT_update_at_offset_immutable(self):
4202 file_contents = "Test file" * 100000
4203 d = self.PUT("/uri", file_contents)
4205 self.filecap = filecap
4206 d.addCallback(_then)
4207 d.addCallback(lambda ignored:
4208 self.shouldHTTPError("PUT_update_at_offset_immutable",
4212 "/uri/%s?offset=50" % self.filecap,
4217 def test_bad_method(self):
4218 url = self.webish_url + self.public_url + "/foo/bar.txt"
4219 d = self.shouldHTTPError("bad_method",
4220 501, "Not Implemented",
4221 "I don't know how to treat a BOGUS request.",
4222 client.getPage, url, method="BOGUS")
4225 def test_short_url(self):
4226 url = self.webish_url + "/uri"
4227 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4228 "I don't know how to treat a DELETE request.",
4229 client.getPage, url, method="DELETE")
4232 def test_ophandle_bad(self):
4233 url = self.webish_url + "/operations/bogus?t=status"
4234 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4235 "unknown/expired handle 'bogus'",
4236 client.getPage, url)
4239 def test_ophandle_cancel(self):
4240 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4241 followRedirect=True)
4242 d.addCallback(lambda ignored:
4243 self.GET("/operations/128?t=status&output=JSON"))
4245 data = simplejson.loads(res)
4246 self.failUnless("finished" in data, res)
4247 monitor = self.ws.root.child_operations.handles["128"][0]
4248 d = self.POST("/operations/128?t=cancel&output=JSON")
4250 data = simplejson.loads(res)
4251 self.failUnless("finished" in data, res)
4252 # t=cancel causes the handle to be forgotten
4253 self.failUnless(monitor.is_cancelled())
4254 d.addCallback(_check2)
4256 d.addCallback(_check1)
4257 d.addCallback(lambda ignored:
4258 self.shouldHTTPError("ophandle_cancel",
4259 404, "404 Not Found",
4260 "unknown/expired handle '128'",
4262 "/operations/128?t=status&output=JSON"))
4265 def test_ophandle_retainfor(self):
4266 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4267 followRedirect=True)
4268 d.addCallback(lambda ignored:
4269 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4271 data = simplejson.loads(res)
4272 self.failUnless("finished" in data, res)
4273 d.addCallback(_check1)
4274 # the retain-for=0 will cause the handle to be expired very soon
4275 d.addCallback(lambda ign:
4276 self.clock.advance(2.0))
4277 d.addCallback(lambda ignored:
4278 self.shouldHTTPError("ophandle_retainfor",
4279 404, "404 Not Found",
4280 "unknown/expired handle '129'",
4282 "/operations/129?t=status&output=JSON"))
4285 def test_ophandle_release_after_complete(self):
4286 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4287 followRedirect=True)
4288 d.addCallback(self.wait_for_operation, "130")
4289 d.addCallback(lambda ignored:
4290 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4291 # the release-after-complete=true will cause the handle to be expired
4292 d.addCallback(lambda ignored:
4293 self.shouldHTTPError("ophandle_release_after_complete",
4294 404, "404 Not Found",
4295 "unknown/expired handle '130'",
4297 "/operations/130?t=status&output=JSON"))
4300 def test_uncollected_ophandle_expiration(self):
4301 # uncollected ophandles should expire after 4 days
4302 def _make_uncollected_ophandle(ophandle):
4303 d = self.POST(self.public_url +
4304 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4305 followRedirect=False)
4306 # When we start the operation, the webapi server will want
4307 # to redirect us to the page for the ophandle, so we get
4308 # confirmation that the operation has started. If the
4309 # manifest operation has finished by the time we get there,
4310 # following that redirect (by setting followRedirect=True
4311 # above) has the side effect of collecting the ophandle that
4312 # we've just created, which means that we can't use the
4313 # ophandle to test the uncollected timeout anymore. So,
4314 # instead, catch the 302 here and don't follow it.
4315 d.addBoth(self.should302, "uncollected_ophandle_creation")
4317 # Create an ophandle, don't collect it, then advance the clock by
4318 # 4 days - 1 second and make sure that the ophandle is still there.
4319 d = _make_uncollected_ophandle(131)
4320 d.addCallback(lambda ign:
4321 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4322 d.addCallback(lambda ign:
4323 self.GET("/operations/131?t=status&output=JSON"))
4325 data = simplejson.loads(res)
4326 self.failUnless("finished" in data, res)
4327 d.addCallback(_check1)
4328 # Create an ophandle, don't collect it, then try to collect it
4329 # after 4 days. It should be gone.
4330 d.addCallback(lambda ign:
4331 _make_uncollected_ophandle(132))
4332 d.addCallback(lambda ign:
4333 self.clock.advance(96*60*60))
4334 d.addCallback(lambda ign:
4335 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4336 404, "404 Not Found",
4337 "unknown/expired handle '132'",
4339 "/operations/132?t=status&output=JSON"))
4342 def test_collected_ophandle_expiration(self):
4343 # collected ophandles should expire after 1 day
4344 def _make_collected_ophandle(ophandle):
4345 d = self.POST(self.public_url +
4346 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4347 followRedirect=True)
4348 # By following the initial redirect, we collect the ophandle
4349 # we've just created.
4351 # Create a collected ophandle, then collect it after 23 hours
4352 # and 59 seconds to make sure that it is still there.
4353 d = _make_collected_ophandle(133)
4354 d.addCallback(lambda ign:
4355 self.clock.advance((24*60*60) - 1))
4356 d.addCallback(lambda ign:
4357 self.GET("/operations/133?t=status&output=JSON"))
4359 data = simplejson.loads(res)
4360 self.failUnless("finished" in data, res)
4361 d.addCallback(_check1)
4362 # Create another uncollected ophandle, then try to collect it
4363 # after 24 hours to make sure that it is gone.
4364 d.addCallback(lambda ign:
4365 _make_collected_ophandle(134))
4366 d.addCallback(lambda ign:
4367 self.clock.advance(24*60*60))
4368 d.addCallback(lambda ign:
4369 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4370 404, "404 Not Found",
4371 "unknown/expired handle '134'",
4373 "/operations/134?t=status&output=JSON"))
4376 def test_incident(self):
4377 d = self.POST("/report_incident", details="eek")
4379 self.failIfIn("<html>", res)
4380 self.failUnlessIn("Thank you for your report!", res)
4381 d.addCallback(_done)
4384 def test_static(self):
4385 webdir = os.path.join(self.staticdir, "subdir")
4386 fileutil.make_dirs(webdir)
4387 f = open(os.path.join(webdir, "hello.txt"), "wb")
4391 d = self.GET("/static/subdir/hello.txt")
4393 self.failUnlessReallyEqual(res, "hello")
4394 d.addCallback(_check)
4398 class IntroducerWeb(unittest.TestCase):
4403 d = defer.succeed(None)
4405 d.addCallback(lambda ign: self.node.stopService())
4406 d.addCallback(flushEventualQueue)
4409 def test_welcome(self):
4410 basedir = "web.IntroducerWeb.test_welcome"
4412 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4413 self.node = IntroducerNode(basedir)
4414 self.ws = self.node.getServiceNamed("webish")
4416 d = fireEventually(None)
4417 d.addCallback(lambda ign: self.node.startService())
4418 d.addCallback(lambda ign: self.node.when_tub_ready())
4420 d.addCallback(lambda ign: self.GET("/"))
4422 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4423 self.failUnlessIn(FAVICON_MARKUP, res)
4424 d.addCallback(_check)
4427 def GET(self, urlpath, followRedirect=False, return_response=False,
4429 # if return_response=True, this fires with (data, statuscode,
4430 # respheaders) instead of just data.
4431 assert not isinstance(urlpath, unicode)
4432 url = self.ws.getURL().rstrip('/') + urlpath
4433 factory = HTTPClientGETFactory(url, method="GET",
4434 followRedirect=followRedirect, **kwargs)
4435 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4436 d = factory.deferred
4437 def _got_data(data):
4438 return (data, factory.status, factory.response_headers)
4440 d.addCallback(_got_data)
4441 return factory.deferred
4444 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4445 def test_load_file(self):
4446 # This will raise an exception unless a well-formed XML file is found under that name.
4447 common.getxmlfile('directory.xhtml').load()
4449 def test_parse_replace_arg(self):
4450 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4451 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4452 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4454 self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4456 def test_abbreviate_time(self):
4457 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4458 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4459 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4460 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4461 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4462 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4464 def test_compute_rate(self):
4465 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4466 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4467 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4468 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4469 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4470 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4471 self.shouldFail(AssertionError, "test_compute_rate", "",
4472 common.compute_rate, -100, 10)
4473 self.shouldFail(AssertionError, "test_compute_rate", "",
4474 common.compute_rate, 100, -10)
4477 rate = common.compute_rate(10*1000*1000, 1)
4478 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4480 def test_abbreviate_rate(self):
4481 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4482 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4483 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4484 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4486 def test_abbreviate_size(self):
4487 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4488 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4489 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4490 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4491 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4493 def test_plural(self):
4495 return "%d second%s" % (s, status.plural(s))
4496 self.failUnlessReallyEqual(convert(0), "0 seconds")
4497 self.failUnlessReallyEqual(convert(1), "1 second")
4498 self.failUnlessReallyEqual(convert(2), "2 seconds")
4500 return "has share%s: %s" % (status.plural(s), ",".join(s))
4501 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4502 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4503 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4506 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4508 def CHECK(self, ign, which, args, clientnum=0):
4509 fileurl = self.fileurls[which]
4510 url = fileurl + "?" + args
4511 return self.GET(url, method="POST", clientnum=clientnum)
4513 def test_filecheck(self):
4514 self.basedir = "web/Grid/filecheck"
4516 c0 = self.g.clients[0]
4519 d = c0.upload(upload.Data(DATA, convergence=""))
4520 def _stash_uri(ur, which):
4521 self.uris[which] = ur.get_uri()
4522 d.addCallback(_stash_uri, "good")
4523 d.addCallback(lambda ign:
4524 c0.upload(upload.Data(DATA+"1", convergence="")))
4525 d.addCallback(_stash_uri, "sick")
4526 d.addCallback(lambda ign:
4527 c0.upload(upload.Data(DATA+"2", convergence="")))
4528 d.addCallback(_stash_uri, "dead")
4529 def _stash_mutable_uri(n, which):
4530 self.uris[which] = n.get_uri()
4531 assert isinstance(self.uris[which], str)
4532 d.addCallback(lambda ign:
4533 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4534 d.addCallback(_stash_mutable_uri, "corrupt")
4535 d.addCallback(lambda ign:
4536 c0.upload(upload.Data("literal", convergence="")))
4537 d.addCallback(_stash_uri, "small")
4538 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4539 d.addCallback(_stash_mutable_uri, "smalldir")
4541 def _compute_fileurls(ignored):
4543 for which in self.uris:
4544 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4545 d.addCallback(_compute_fileurls)
4547 def _clobber_shares(ignored):
4548 good_shares = self.find_uri_shares(self.uris["good"])
4549 self.failUnlessReallyEqual(len(good_shares), 10)
4550 sick_shares = self.find_uri_shares(self.uris["sick"])
4551 os.unlink(sick_shares[0][2])
4552 dead_shares = self.find_uri_shares(self.uris["dead"])
4553 for i in range(1, 10):
4554 os.unlink(dead_shares[i][2])
4555 c_shares = self.find_uri_shares(self.uris["corrupt"])
4556 cso = CorruptShareOptions()
4557 cso.stdout = StringIO()
4558 cso.parseOptions([c_shares[0][2]])
4560 d.addCallback(_clobber_shares)
4562 d.addCallback(self.CHECK, "good", "t=check")
4563 def _got_html_good(res):
4564 self.failUnlessIn("Healthy", res)
4565 self.failIfIn("Not Healthy", res)
4566 self.failUnlessIn(FAVICON_MARKUP, res)
4567 d.addCallback(_got_html_good)
4568 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4569 def _got_html_good_return_to(res):
4570 self.failUnlessIn("Healthy", res)
4571 self.failIfIn("Not Healthy", res)
4572 self.failUnlessIn('<a href="somewhere">Return to file', res)
4573 d.addCallback(_got_html_good_return_to)
4574 d.addCallback(self.CHECK, "good", "t=check&output=json")
4575 def _got_json_good(res):
4576 r = simplejson.loads(res)
4577 self.failUnlessEqual(r["summary"], "Healthy")
4578 self.failUnless(r["results"]["healthy"])
4579 self.failIf(r["results"]["needs-rebalancing"])
4580 self.failUnless(r["results"]["recoverable"])
4581 d.addCallback(_got_json_good)
4583 d.addCallback(self.CHECK, "small", "t=check")
4584 def _got_html_small(res):
4585 self.failUnlessIn("Literal files are always healthy", res)
4586 self.failIfIn("Not Healthy", res)
4587 d.addCallback(_got_html_small)
4588 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4589 def _got_html_small_return_to(res):
4590 self.failUnlessIn("Literal files are always healthy", res)
4591 self.failIfIn("Not Healthy", res)
4592 self.failUnlessIn('<a href="somewhere">Return to file', res)
4593 d.addCallback(_got_html_small_return_to)
4594 d.addCallback(self.CHECK, "small", "t=check&output=json")
4595 def _got_json_small(res):
4596 r = simplejson.loads(res)
4597 self.failUnlessEqual(r["storage-index"], "")
4598 self.failUnless(r["results"]["healthy"])
4599 d.addCallback(_got_json_small)
4601 d.addCallback(self.CHECK, "smalldir", "t=check")
4602 def _got_html_smalldir(res):
4603 self.failUnlessIn("Literal files are always healthy", res)
4604 self.failIfIn("Not Healthy", res)
4605 d.addCallback(_got_html_smalldir)
4606 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4607 def _got_json_smalldir(res):
4608 r = simplejson.loads(res)
4609 self.failUnlessEqual(r["storage-index"], "")
4610 self.failUnless(r["results"]["healthy"])
4611 d.addCallback(_got_json_smalldir)
4613 d.addCallback(self.CHECK, "sick", "t=check")
4614 def _got_html_sick(res):
4615 self.failUnlessIn("Not Healthy", res)
4616 d.addCallback(_got_html_sick)
4617 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4618 def _got_json_sick(res):
4619 r = simplejson.loads(res)
4620 self.failUnlessEqual(r["summary"],
4621 "Not Healthy: 9 shares (enc 3-of-10)")
4622 self.failIf(r["results"]["healthy"])
4623 self.failIf(r["results"]["needs-rebalancing"])
4624 self.failUnless(r["results"]["recoverable"])
4625 d.addCallback(_got_json_sick)
4627 d.addCallback(self.CHECK, "dead", "t=check")
4628 def _got_html_dead(res):
4629 self.failUnlessIn("Not Healthy", res)
4630 d.addCallback(_got_html_dead)
4631 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4632 def _got_json_dead(res):
4633 r = simplejson.loads(res)
4634 self.failUnlessEqual(r["summary"],
4635 "Not Healthy: 1 shares (enc 3-of-10)")
4636 self.failIf(r["results"]["healthy"])
4637 self.failIf(r["results"]["needs-rebalancing"])
4638 self.failIf(r["results"]["recoverable"])
4639 d.addCallback(_got_json_dead)
4641 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4642 def _got_html_corrupt(res):
4643 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4644 d.addCallback(_got_html_corrupt)
4645 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4646 def _got_json_corrupt(res):
4647 r = simplejson.loads(res)
4648 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4649 self.failIf(r["results"]["healthy"])
4650 self.failUnless(r["results"]["recoverable"])
4651 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4652 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4653 d.addCallback(_got_json_corrupt)
4655 d.addErrback(self.explain_web_error)
4658 def test_repair_html(self):
4659 self.basedir = "web/Grid/repair_html"
4661 c0 = self.g.clients[0]
4664 d = c0.upload(upload.Data(DATA, convergence=""))
4665 def _stash_uri(ur, which):
4666 self.uris[which] = ur.get_uri()
4667 d.addCallback(_stash_uri, "good")
4668 d.addCallback(lambda ign:
4669 c0.upload(upload.Data(DATA+"1", convergence="")))
4670 d.addCallback(_stash_uri, "sick")
4671 d.addCallback(lambda ign:
4672 c0.upload(upload.Data(DATA+"2", convergence="")))
4673 d.addCallback(_stash_uri, "dead")
4674 def _stash_mutable_uri(n, which):
4675 self.uris[which] = n.get_uri()
4676 assert isinstance(self.uris[which], str)
4677 d.addCallback(lambda ign:
4678 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4679 d.addCallback(_stash_mutable_uri, "corrupt")
4681 def _compute_fileurls(ignored):
4683 for which in self.uris:
4684 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4685 d.addCallback(_compute_fileurls)
4687 def _clobber_shares(ignored):
4688 good_shares = self.find_uri_shares(self.uris["good"])
4689 self.failUnlessReallyEqual(len(good_shares), 10)
4690 sick_shares = self.find_uri_shares(self.uris["sick"])
4691 os.unlink(sick_shares[0][2])
4692 dead_shares = self.find_uri_shares(self.uris["dead"])
4693 for i in range(1, 10):
4694 os.unlink(dead_shares[i][2])
4695 c_shares = self.find_uri_shares(self.uris["corrupt"])
4696 cso = CorruptShareOptions()
4697 cso.stdout = StringIO()
4698 cso.parseOptions([c_shares[0][2]])
4700 d.addCallback(_clobber_shares)
4702 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4703 def _got_html_good(res):
4704 self.failUnlessIn("Healthy", res)
4705 self.failIfIn("Not Healthy", res)
4706 self.failUnlessIn("No repair necessary", res)
4707 self.failUnlessIn(FAVICON_MARKUP, res)
4708 d.addCallback(_got_html_good)
4710 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4711 def _got_html_sick(res):
4712 self.failUnlessIn("Healthy : healthy", res)
4713 self.failIfIn("Not Healthy", res)
4714 self.failUnlessIn("Repair successful", res)
4715 d.addCallback(_got_html_sick)
4717 # repair of a dead file will fail, of course, but it isn't yet
4718 # clear how this should be reported. Right now it shows up as
4721 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4722 #def _got_html_dead(res):
4724 # self.failUnlessIn("Healthy : healthy", res)
4725 # self.failIfIn("Not Healthy", res)
4726 # self.failUnlessIn("No repair necessary", res)
4727 #d.addCallback(_got_html_dead)
4729 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4730 def _got_html_corrupt(res):
4731 self.failUnlessIn("Healthy : Healthy", res)
4732 self.failIfIn("Not Healthy", res)
4733 self.failUnlessIn("Repair successful", res)
4734 d.addCallback(_got_html_corrupt)
4736 d.addErrback(self.explain_web_error)
4739 def test_repair_json(self):
4740 self.basedir = "web/Grid/repair_json"
4742 c0 = self.g.clients[0]
4745 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4746 def _stash_uri(ur, which):
4747 self.uris[which] = ur.get_uri()
4748 d.addCallback(_stash_uri, "sick")
4750 def _compute_fileurls(ignored):
4752 for which in self.uris:
4753 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4754 d.addCallback(_compute_fileurls)
4756 def _clobber_shares(ignored):
4757 sick_shares = self.find_uri_shares(self.uris["sick"])
4758 os.unlink(sick_shares[0][2])
4759 d.addCallback(_clobber_shares)
4761 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4762 def _got_json_sick(res):
4763 r = simplejson.loads(res)
4764 self.failUnlessReallyEqual(r["repair-attempted"], True)
4765 self.failUnlessReallyEqual(r["repair-successful"], True)
4766 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4767 "Not Healthy: 9 shares (enc 3-of-10)")
4768 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4769 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4770 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4771 d.addCallback(_got_json_sick)
4773 d.addErrback(self.explain_web_error)
4776 def test_unknown(self, immutable=False):
4777 self.basedir = "web/Grid/unknown"
4779 self.basedir = "web/Grid/unknown-immutable"
4782 c0 = self.g.clients[0]
4786 # the future cap format may contain slashes, which must be tolerated
4787 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4791 name = u"future-imm"
4792 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4793 d = c0.create_immutable_dirnode({name: (future_node, {})})
4796 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4797 d = c0.create_dirnode()
4799 def _stash_root_and_create_file(n):
4801 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4802 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4804 return self.rootnode.set_node(name, future_node)
4805 d.addCallback(_stash_root_and_create_file)
4807 # make sure directory listing tolerates unknown nodes
4808 d.addCallback(lambda ign: self.GET(self.rooturl))
4809 def _check_directory_html(res, expected_type_suffix):
4810 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4811 '<td>%s</td>' % (expected_type_suffix, str(name)),
4813 self.failUnless(re.search(pattern, res), res)
4814 # find the More Info link for name, should be relative
4815 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4816 info_url = mo.group(1)
4817 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4819 d.addCallback(_check_directory_html, "-IMM")
4821 d.addCallback(_check_directory_html, "")
4823 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4824 def _check_directory_json(res, expect_rw_uri):
4825 data = simplejson.loads(res)
4826 self.failUnlessEqual(data[0], "dirnode")
4827 f = data[1]["children"][name]
4828 self.failUnlessEqual(f[0], "unknown")
4830 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4832 self.failIfIn("rw_uri", f[1])
4834 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4836 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4837 self.failUnlessIn("metadata", f[1])
4838 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4840 def _check_info(res, expect_rw_uri, expect_ro_uri):
4841 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4843 self.failUnlessIn(unknown_rwcap, res)
4846 self.failUnlessIn(unknown_immcap, res)
4848 self.failUnlessIn(unknown_rocap, res)
4850 self.failIfIn(unknown_rocap, res)
4851 self.failIfIn("Raw data as", res)
4852 self.failIfIn("Directory writecap", res)
4853 self.failIfIn("Checker Operations", res)
4854 self.failIfIn("Mutable File Operations", res)
4855 self.failIfIn("Directory Operations", res)
4857 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4858 # why they fail. Possibly related to ticket #922.
4860 d.addCallback(lambda ign: self.GET(expected_info_url))
4861 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4862 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4863 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4865 def _check_json(res, expect_rw_uri):
4866 data = simplejson.loads(res)
4867 self.failUnlessEqual(data[0], "unknown")
4869 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4871 self.failIfIn("rw_uri", data[1])
4874 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4875 self.failUnlessReallyEqual(data[1]["mutable"], False)
4877 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4878 self.failUnlessReallyEqual(data[1]["mutable"], True)
4880 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4881 self.failIfIn("mutable", data[1])
4883 # TODO: check metadata contents
4884 self.failUnlessIn("metadata", data[1])
4886 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4887 d.addCallback(_check_json, expect_rw_uri=not immutable)
4889 # and make sure that a read-only version of the directory can be
4890 # rendered too. This version will not have unknown_rwcap, whether
4891 # or not future_node was immutable.
4892 d.addCallback(lambda ign: self.GET(self.rourl))
4894 d.addCallback(_check_directory_html, "-IMM")
4896 d.addCallback(_check_directory_html, "-RO")
4898 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4899 d.addCallback(_check_directory_json, expect_rw_uri=False)
4901 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4902 d.addCallback(_check_json, expect_rw_uri=False)
4904 # TODO: check that getting t=info from the Info link in the ro directory
4905 # works, and does not include the writecap URI.
4908 def test_immutable_unknown(self):
4909 return self.test_unknown(immutable=True)
4911 def test_mutant_dirnodes_are_omitted(self):
4912 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4915 c = self.g.clients[0]
4920 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4921 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4922 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4924 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4925 # test the dirnode and web layers separately.
4927 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4928 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4929 # When the directory is read, the mutants should be silently disposed of, leaving
4930 # their lonely sibling.
4931 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4932 # because immutable directories don't have a writecap and therefore that field
4933 # isn't (and can't be) decrypted.
4934 # TODO: The field still exists in the netstring. Technically we should check what
4935 # happens if something is put there (_unpack_contents should raise ValueError),
4936 # but that can wait.
4938 lonely_child = nm.create_from_cap(lonely_uri)
4939 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4940 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4942 def _by_hook_or_by_crook():
4944 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4945 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4947 mutant_write_in_ro_child.get_write_uri = lambda: None
4948 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4950 kids = {u"lonely": (lonely_child, {}),
4951 u"ro": (mutant_ro_child, {}),
4952 u"write-in-ro": (mutant_write_in_ro_child, {}),
4954 d = c.create_immutable_dirnode(kids)
4957 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4958 self.failIf(dn.is_mutable())
4959 self.failUnless(dn.is_readonly())
4960 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4961 self.failIf(hasattr(dn._node, 'get_writekey'))
4963 self.failUnlessIn("RO-IMM", rep)
4965 self.failUnlessIn("CHK", cap.to_string())
4968 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4969 return download_to_data(dn._node)
4970 d.addCallback(_created)
4972 def _check_data(data):
4973 # Decode the netstring representation of the directory to check that all children
4974 # are present. This is a bit of an abstraction violation, but there's not really
4975 # any other way to do it given that the real DirectoryNode._unpack_contents would
4976 # strip the mutant children out (which is what we're trying to test, later).
4979 while position < len(data):
4980 entries, position = split_netstring(data, 1, position)
4982 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4983 name = name_utf8.decode("utf-8")
4984 self.failUnlessEqual(rwcapdata, "")
4985 self.failUnlessIn(name, kids)
4986 (expected_child, ign) = kids[name]
4987 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4990 self.failUnlessReallyEqual(numkids, 3)
4991 return self.rootnode.list()
4992 d.addCallback(_check_data)
4994 # Now when we use the real directory listing code, the mutants should be absent.
4995 def _check_kids(children):
4996 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4997 lonely_node, lonely_metadata = children[u"lonely"]
4999 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5000 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5001 d.addCallback(_check_kids)
5003 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5004 d.addCallback(lambda n: n.list())
5005 d.addCallback(_check_kids) # again with dirnode recreated from cap
5007 # Make sure the lonely child can be listed in HTML...
5008 d.addCallback(lambda ign: self.GET(self.rooturl))
5009 def _check_html(res):
5010 self.failIfIn("URI:SSK", res)
5011 get_lonely = "".join([r'<td>FILE</td>',
5013 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5015 r'\s+<td align="right">%d</td>' % len("one"),
5017 self.failUnless(re.search(get_lonely, res), res)
5019 # find the More Info link for name, should be relative
5020 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5021 info_url = mo.group(1)
5022 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5023 d.addCallback(_check_html)
5026 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5027 def _check_json(res):
5028 data = simplejson.loads(res)
5029 self.failUnlessEqual(data[0], "dirnode")
5030 listed_children = data[1]["children"]
5031 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5032 ll_type, ll_data = listed_children[u"lonely"]
5033 self.failUnlessEqual(ll_type, "filenode")
5034 self.failIfIn("rw_uri", ll_data)
5035 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5036 d.addCallback(_check_json)
5039 def test_deep_check(self):
5040 self.basedir = "web/Grid/deep_check"
5042 c0 = self.g.clients[0]
5046 d = c0.create_dirnode()
5047 def _stash_root_and_create_file(n):
5049 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5050 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5051 d.addCallback(_stash_root_and_create_file)
5052 def _stash_uri(fn, which):
5053 self.uris[which] = fn.get_uri()
5055 d.addCallback(_stash_uri, "good")
5056 d.addCallback(lambda ign:
5057 self.rootnode.add_file(u"small",
5058 upload.Data("literal",
5060 d.addCallback(_stash_uri, "small")
5061 d.addCallback(lambda ign:
5062 self.rootnode.add_file(u"sick",
5063 upload.Data(DATA+"1",
5065 d.addCallback(_stash_uri, "sick")
5067 # this tests that deep-check and stream-manifest will ignore
5068 # UnknownNode instances. Hopefully this will also cover deep-stats.
5069 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5070 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5072 def _clobber_shares(ignored):
5073 self.delete_shares_numbered(self.uris["sick"], [0,1])
5074 d.addCallback(_clobber_shares)
5082 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5085 units = [simplejson.loads(line)
5086 for line in res.splitlines()
5089 print "response is:", res
5090 print "undecodeable line was '%s'" % line
5092 self.failUnlessReallyEqual(len(units), 5+1)
5093 # should be parent-first
5095 self.failUnlessEqual(u0["path"], [])
5096 self.failUnlessEqual(u0["type"], "directory")
5097 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5098 u0cr = u0["check-results"]
5099 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5101 ugood = [u for u in units
5102 if u["type"] == "file" and u["path"] == [u"good"]][0]
5103 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5104 ugoodcr = ugood["check-results"]
5105 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5108 self.failUnlessEqual(stats["type"], "stats")
5110 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5111 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5112 self.failUnlessReallyEqual(s["count-directories"], 1)
5113 self.failUnlessReallyEqual(s["count-unknown"], 1)
5114 d.addCallback(_done)
5116 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5117 def _check_manifest(res):
5118 self.failUnless(res.endswith("\n"))
5119 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5120 self.failUnlessReallyEqual(len(units), 5+1)
5121 self.failUnlessEqual(units[-1]["type"], "stats")
5123 self.failUnlessEqual(first["path"], [])
5124 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5125 self.failUnlessEqual(first["type"], "directory")
5126 stats = units[-1]["stats"]
5127 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5128 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5129 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5130 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5131 self.failUnlessReallyEqual(stats["count-unknown"], 1)
5132 d.addCallback(_check_manifest)
5134 # now add root/subdir and root/subdir/grandchild, then make subdir
5135 # unrecoverable, then see what happens
5137 d.addCallback(lambda ign:
5138 self.rootnode.create_subdirectory(u"subdir"))
5139 d.addCallback(_stash_uri, "subdir")
5140 d.addCallback(lambda subdir_node:
5141 subdir_node.add_file(u"grandchild",
5142 upload.Data(DATA+"2",
5144 d.addCallback(_stash_uri, "grandchild")
5146 d.addCallback(lambda ign:
5147 self.delete_shares_numbered(self.uris["subdir"],
5155 # root/subdir [unrecoverable]
5156 # root/subdir/grandchild
5158 # how should a streaming-JSON API indicate fatal error?
5159 # answer: emit ERROR: instead of a JSON string
5161 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5162 def _check_broken_manifest(res):
5163 lines = res.splitlines()
5165 for (i,line) in enumerate(lines)
5166 if line.startswith("ERROR:")]
5168 self.fail("no ERROR: in output: %s" % (res,))
5169 first_error = error_lines[0]
5170 error_line = lines[first_error]
5171 error_msg = lines[first_error+1:]
5172 error_msg_s = "\n".join(error_msg) + "\n"
5173 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5175 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5176 units = [simplejson.loads(line) for line in lines[:first_error]]
5177 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5178 last_unit = units[-1]
5179 self.failUnlessEqual(last_unit["path"], ["subdir"])
5180 d.addCallback(_check_broken_manifest)
5182 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5183 def _check_broken_deepcheck(res):
5184 lines = res.splitlines()
5186 for (i,line) in enumerate(lines)
5187 if line.startswith("ERROR:")]
5189 self.fail("no ERROR: in output: %s" % (res,))
5190 first_error = error_lines[0]
5191 error_line = lines[first_error]
5192 error_msg = lines[first_error+1:]
5193 error_msg_s = "\n".join(error_msg) + "\n"
5194 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5196 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5197 units = [simplejson.loads(line) for line in lines[:first_error]]
5198 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5199 last_unit = units[-1]
5200 self.failUnlessEqual(last_unit["path"], ["subdir"])
5201 r = last_unit["check-results"]["results"]
5202 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5203 self.failUnlessReallyEqual(r["count-shares-good"], 1)
5204 self.failUnlessReallyEqual(r["recoverable"], False)
5205 d.addCallback(_check_broken_deepcheck)
5207 d.addErrback(self.explain_web_error)
5210 def test_deep_check_and_repair(self):
5211 self.basedir = "web/Grid/deep_check_and_repair"
5213 c0 = self.g.clients[0]
5217 d = c0.create_dirnode()
5218 def _stash_root_and_create_file(n):
5220 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5221 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5222 d.addCallback(_stash_root_and_create_file)
5223 def _stash_uri(fn, which):
5224 self.uris[which] = fn.get_uri()
5225 d.addCallback(_stash_uri, "good")
5226 d.addCallback(lambda ign:
5227 self.rootnode.add_file(u"small",
5228 upload.Data("literal",
5230 d.addCallback(_stash_uri, "small")
5231 d.addCallback(lambda ign:
5232 self.rootnode.add_file(u"sick",
5233 upload.Data(DATA+"1",
5235 d.addCallback(_stash_uri, "sick")
5236 #d.addCallback(lambda ign:
5237 # self.rootnode.add_file(u"dead",
5238 # upload.Data(DATA+"2",
5240 #d.addCallback(_stash_uri, "dead")
5242 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5243 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5244 #d.addCallback(_stash_uri, "corrupt")
5246 def _clobber_shares(ignored):
5247 good_shares = self.find_uri_shares(self.uris["good"])
5248 self.failUnlessReallyEqual(len(good_shares), 10)
5249 sick_shares = self.find_uri_shares(self.uris["sick"])
5250 os.unlink(sick_shares[0][2])
5251 #dead_shares = self.find_uri_shares(self.uris["dead"])
5252 #for i in range(1, 10):
5253 # os.unlink(dead_shares[i][2])
5255 #c_shares = self.find_uri_shares(self.uris["corrupt"])
5256 #cso = CorruptShareOptions()
5257 #cso.stdout = StringIO()
5258 #cso.parseOptions([c_shares[0][2]])
5260 d.addCallback(_clobber_shares)
5263 # root/good CHK, 10 shares
5265 # root/sick CHK, 9 shares
5267 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5269 units = [simplejson.loads(line)
5270 for line in res.splitlines()
5272 self.failUnlessReallyEqual(len(units), 4+1)
5273 # should be parent-first
5275 self.failUnlessEqual(u0["path"], [])
5276 self.failUnlessEqual(u0["type"], "directory")
5277 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5278 u0crr = u0["check-and-repair-results"]
5279 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5280 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5282 ugood = [u for u in units
5283 if u["type"] == "file" and u["path"] == [u"good"]][0]
5284 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5285 ugoodcrr = ugood["check-and-repair-results"]
5286 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5287 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5289 usick = [u for u in units
5290 if u["type"] == "file" and u["path"] == [u"sick"]][0]
5291 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5292 usickcrr = usick["check-and-repair-results"]
5293 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5294 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5295 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5296 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5299 self.failUnlessEqual(stats["type"], "stats")
5301 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5302 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5303 self.failUnlessReallyEqual(s["count-directories"], 1)
5304 d.addCallback(_done)
5306 d.addErrback(self.explain_web_error)
5309 def _count_leases(self, ignored, which):
5310 u = self.uris[which]
5311 shares = self.find_uri_shares(u)
5313 for shnum, serverid, fn in shares:
5314 sf = get_share_file(fn)
5315 num_leases = len(list(sf.get_leases()))
5316 lease_counts.append( (fn, num_leases) )
5319 def _assert_leasecount(self, lease_counts, expected):
5320 for (fn, num_leases) in lease_counts:
5321 if num_leases != expected:
5322 self.fail("expected %d leases, have %d, on %s" %
5323 (expected, num_leases, fn))
5325 def test_add_lease(self):
5326 self.basedir = "web/Grid/add_lease"
5327 self.set_up_grid(num_clients=2)
5328 c0 = self.g.clients[0]
5331 d = c0.upload(upload.Data(DATA, convergence=""))
5332 def _stash_uri(ur, which):
5333 self.uris[which] = ur.get_uri()
5334 d.addCallback(_stash_uri, "one")
5335 d.addCallback(lambda ign:
5336 c0.upload(upload.Data(DATA+"1", convergence="")))
5337 d.addCallback(_stash_uri, "two")
5338 def _stash_mutable_uri(n, which):
5339 self.uris[which] = n.get_uri()
5340 assert isinstance(self.uris[which], str)
5341 d.addCallback(lambda ign:
5342 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5343 d.addCallback(_stash_mutable_uri, "mutable")
5345 def _compute_fileurls(ignored):
5347 for which in self.uris:
5348 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5349 d.addCallback(_compute_fileurls)
5351 d.addCallback(self._count_leases, "one")
5352 d.addCallback(self._assert_leasecount, 1)
5353 d.addCallback(self._count_leases, "two")
5354 d.addCallback(self._assert_leasecount, 1)
5355 d.addCallback(self._count_leases, "mutable")
5356 d.addCallback(self._assert_leasecount, 1)
5358 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5359 def _got_html_good(res):
5360 self.failUnlessIn("Healthy", res)
5361 self.failIfIn("Not Healthy", res)
5362 d.addCallback(_got_html_good)
5364 d.addCallback(self._count_leases, "one")
5365 d.addCallback(self._assert_leasecount, 1)
5366 d.addCallback(self._count_leases, "two")
5367 d.addCallback(self._assert_leasecount, 1)
5368 d.addCallback(self._count_leases, "mutable")
5369 d.addCallback(self._assert_leasecount, 1)
5371 # this CHECK uses the original client, which uses the same
5372 # lease-secrets, so it will just renew the original lease
5373 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5374 d.addCallback(_got_html_good)
5376 d.addCallback(self._count_leases, "one")
5377 d.addCallback(self._assert_leasecount, 1)
5378 d.addCallback(self._count_leases, "two")
5379 d.addCallback(self._assert_leasecount, 1)
5380 d.addCallback(self._count_leases, "mutable")
5381 d.addCallback(self._assert_leasecount, 1)
5383 # this CHECK uses an alternate client, which adds a second lease
5384 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5385 d.addCallback(_got_html_good)
5387 d.addCallback(self._count_leases, "one")
5388 d.addCallback(self._assert_leasecount, 2)
5389 d.addCallback(self._count_leases, "two")
5390 d.addCallback(self._assert_leasecount, 1)
5391 d.addCallback(self._count_leases, "mutable")
5392 d.addCallback(self._assert_leasecount, 1)
5394 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5395 d.addCallback(_got_html_good)
5397 d.addCallback(self._count_leases, "one")
5398 d.addCallback(self._assert_leasecount, 2)
5399 d.addCallback(self._count_leases, "two")
5400 d.addCallback(self._assert_leasecount, 1)
5401 d.addCallback(self._count_leases, "mutable")
5402 d.addCallback(self._assert_leasecount, 1)
5404 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5406 d.addCallback(_got_html_good)
5408 d.addCallback(self._count_leases, "one")
5409 d.addCallback(self._assert_leasecount, 2)
5410 d.addCallback(self._count_leases, "two")
5411 d.addCallback(self._assert_leasecount, 1)
5412 d.addCallback(self._count_leases, "mutable")
5413 d.addCallback(self._assert_leasecount, 2)
5415 d.addErrback(self.explain_web_error)
5418 def test_deep_add_lease(self):
5419 self.basedir = "web/Grid/deep_add_lease"
5420 self.set_up_grid(num_clients=2)
5421 c0 = self.g.clients[0]
5425 d = c0.create_dirnode()
5426 def _stash_root_and_create_file(n):
5428 self.uris["root"] = n.get_uri()
5429 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5430 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5431 d.addCallback(_stash_root_and_create_file)
5432 def _stash_uri(fn, which):
5433 self.uris[which] = fn.get_uri()
5434 d.addCallback(_stash_uri, "one")
5435 d.addCallback(lambda ign:
5436 self.rootnode.add_file(u"small",
5437 upload.Data("literal",
5439 d.addCallback(_stash_uri, "small")
5441 d.addCallback(lambda ign:
5442 c0.create_mutable_file(publish.MutableData("mutable")))
5443 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5444 d.addCallback(_stash_uri, "mutable")
5446 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5448 units = [simplejson.loads(line)
5449 for line in res.splitlines()
5451 # root, one, small, mutable, stats
5452 self.failUnlessReallyEqual(len(units), 4+1)
5453 d.addCallback(_done)
5455 d.addCallback(self._count_leases, "root")
5456 d.addCallback(self._assert_leasecount, 1)
5457 d.addCallback(self._count_leases, "one")
5458 d.addCallback(self._assert_leasecount, 1)
5459 d.addCallback(self._count_leases, "mutable")
5460 d.addCallback(self._assert_leasecount, 1)
5462 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5463 d.addCallback(_done)
5465 d.addCallback(self._count_leases, "root")
5466 d.addCallback(self._assert_leasecount, 1)
5467 d.addCallback(self._count_leases, "one")
5468 d.addCallback(self._assert_leasecount, 1)
5469 d.addCallback(self._count_leases, "mutable")
5470 d.addCallback(self._assert_leasecount, 1)
5472 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5474 d.addCallback(_done)
5476 d.addCallback(self._count_leases, "root")
5477 d.addCallback(self._assert_leasecount, 2)
5478 d.addCallback(self._count_leases, "one")
5479 d.addCallback(self._assert_leasecount, 2)
5480 d.addCallback(self._count_leases, "mutable")
5481 d.addCallback(self._assert_leasecount, 2)
5483 d.addErrback(self.explain_web_error)
5487 def test_exceptions(self):
5488 self.basedir = "web/Grid/exceptions"
5489 self.set_up_grid(num_clients=1, num_servers=2)
5490 c0 = self.g.clients[0]
5491 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5494 d = c0.create_dirnode()
5496 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5497 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5499 d.addCallback(_stash_root)
5500 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5502 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5503 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5505 u = uri.from_string(ur.get_uri())
5506 u.key = testutil.flip_bit(u.key, 0)
5507 baduri = u.to_string()
5508 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5509 d.addCallback(_stash_bad)
5510 d.addCallback(lambda ign: c0.create_dirnode())
5511 def _mangle_dirnode_1share(n):
5513 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5514 self.fileurls["dir-1share-json"] = url + "?t=json"
5515 self.delete_shares_numbered(u, range(1,10))
5516 d.addCallback(_mangle_dirnode_1share)
5517 d.addCallback(lambda ign: c0.create_dirnode())
5518 def _mangle_dirnode_0share(n):
5520 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5521 self.fileurls["dir-0share-json"] = url + "?t=json"
5522 self.delete_shares_numbered(u, range(0,10))
5523 d.addCallback(_mangle_dirnode_0share)
5525 # NotEnoughSharesError should be reported sensibly, with a
5526 # text/plain explanation of the problem, and perhaps some
5527 # information on which shares *could* be found.
5529 d.addCallback(lambda ignored:
5530 self.shouldHTTPError("GET unrecoverable",
5531 410, "Gone", "NoSharesError",
5532 self.GET, self.fileurls["0shares"]))
5533 def _check_zero_shares(body):
5534 self.failIfIn("<html>", body)
5535 body = " ".join(body.strip().split())
5536 exp = ("NoSharesError: no shares could be found. "
5537 "Zero shares usually indicates a corrupt URI, or that "
5538 "no servers were connected, but it might also indicate "
5539 "severe corruption. You should perform a filecheck on "
5540 "this object to learn more. The full error message is: "
5541 "no shares (need 3). Last failure: None")
5542 self.failUnlessReallyEqual(exp, body)
5543 d.addCallback(_check_zero_shares)
5546 d.addCallback(lambda ignored:
5547 self.shouldHTTPError("GET 1share",
5548 410, "Gone", "NotEnoughSharesError",
5549 self.GET, self.fileurls["1share"]))
5550 def _check_one_share(body):
5551 self.failIfIn("<html>", body)
5552 body = " ".join(body.strip().split())
5553 msgbase = ("NotEnoughSharesError: This indicates that some "
5554 "servers were unavailable, or that shares have been "
5555 "lost to server departure, hard drive failure, or disk "
5556 "corruption. You should perform a filecheck on "
5557 "this object to learn more. The full error message is:"
5559 msg1 = msgbase + (" ran out of shares:"
5562 " overdue= unused= need 3. Last failure: None")
5563 msg2 = msgbase + (" ran out of shares:"
5565 " pending=Share(sh0-on-xgru5)"
5566 " overdue= unused= need 3. Last failure: None")
5567 self.failUnless(body == msg1 or body == msg2, body)
5568 d.addCallback(_check_one_share)
5570 d.addCallback(lambda ignored:
5571 self.shouldHTTPError("GET imaginary",
5572 404, "Not Found", None,
5573 self.GET, self.fileurls["imaginary"]))
5574 def _missing_child(body):
5575 self.failUnlessIn("No such child: imaginary", body)
5576 d.addCallback(_missing_child)
5578 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5579 def _check_0shares_dir_html(body):
5580 self.failUnlessIn("<html>", body)
5581 # we should see the regular page, but without the child table or
5583 body = " ".join(body.strip().split())
5584 self.failUnlessIn('href="?t=info">More info on this directory',
5586 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5587 "could not be retrieved, because there were insufficient "
5588 "good shares. This might indicate that no servers were "
5589 "connected, insufficient servers were connected, the URI "
5590 "was corrupt, or that shares have been lost due to server "
5591 "departure, hard drive failure, or disk corruption. You "
5592 "should perform a filecheck on this object to learn more.")
5593 self.failUnlessIn(exp, body)
5594 self.failUnlessIn("No upload forms: directory is unreadable", body)
5595 d.addCallback(_check_0shares_dir_html)
5597 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5598 def _check_1shares_dir_html(body):
5599 # at some point, we'll split UnrecoverableFileError into 0-shares
5600 # and some-shares like we did for immutable files (since there
5601 # are different sorts of advice to offer in each case). For now,
5602 # they present the same way.
5603 self.failUnlessIn("<html>", body)
5604 body = " ".join(body.strip().split())
5605 self.failUnlessIn('href="?t=info">More info on this directory',
5607 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5608 "could not be retrieved, because there were insufficient "
5609 "good shares. This might indicate that no servers were "
5610 "connected, insufficient servers were connected, the URI "
5611 "was corrupt, or that shares have been lost due to server "
5612 "departure, hard drive failure, or disk corruption. You "
5613 "should perform a filecheck on this object to learn more.")
5614 self.failUnlessIn(exp, body)
5615 self.failUnlessIn("No upload forms: directory is unreadable", body)
5616 d.addCallback(_check_1shares_dir_html)
5618 d.addCallback(lambda ignored:
5619 self.shouldHTTPError("GET dir-0share-json",
5620 410, "Gone", "UnrecoverableFileError",
5622 self.fileurls["dir-0share-json"]))
5623 def _check_unrecoverable_file(body):
5624 self.failIfIn("<html>", body)
5625 body = " ".join(body.strip().split())
5626 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5627 "could not be retrieved, because there were insufficient "
5628 "good shares. This might indicate that no servers were "
5629 "connected, insufficient servers were connected, the URI "
5630 "was corrupt, or that shares have been lost due to server "
5631 "departure, hard drive failure, or disk corruption. You "
5632 "should perform a filecheck on this object to learn more.")
5633 self.failUnlessReallyEqual(exp, body)
5634 d.addCallback(_check_unrecoverable_file)
5636 d.addCallback(lambda ignored:
5637 self.shouldHTTPError("GET dir-1share-json",
5638 410, "Gone", "UnrecoverableFileError",
5640 self.fileurls["dir-1share-json"]))
5641 d.addCallback(_check_unrecoverable_file)
5643 d.addCallback(lambda ignored:
5644 self.shouldHTTPError("GET imaginary",
5645 404, "Not Found", None,
5646 self.GET, self.fileurls["imaginary"]))
5648 # attach a webapi child that throws a random error, to test how it
5650 w = c0.getServiceNamed("webish")
5651 w.root.putChild("ERRORBOOM", ErrorBoom())
5653 # "Accept: */*" : should get a text/html stack trace
5654 # "Accept: text/plain" : should get a text/plain stack trace
5655 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5656 # no Accept header: should get a text/html stack trace
5658 d.addCallback(lambda ignored:
5659 self.shouldHTTPError("GET errorboom_html",
5660 500, "Internal Server Error", None,
5661 self.GET, "ERRORBOOM",
5662 headers={"accept": "*/*"}))
5663 def _internal_error_html1(body):
5664 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5665 d.addCallback(_internal_error_html1)
5667 d.addCallback(lambda ignored:
5668 self.shouldHTTPError("GET errorboom_text",
5669 500, "Internal Server Error", None,
5670 self.GET, "ERRORBOOM",
5671 headers={"accept": "text/plain"}))
5672 def _internal_error_text2(body):
5673 self.failIfIn("<html>", body)
5674 self.failUnless(body.startswith("Traceback "), body)
5675 d.addCallback(_internal_error_text2)
5677 CLI_accepts = "text/plain, application/octet-stream"
5678 d.addCallback(lambda ignored:
5679 self.shouldHTTPError("GET errorboom_text",
5680 500, "Internal Server Error", None,
5681 self.GET, "ERRORBOOM",
5682 headers={"accept": CLI_accepts}))
5683 def _internal_error_text3(body):
5684 self.failIfIn("<html>", body)
5685 self.failUnless(body.startswith("Traceback "), body)
5686 d.addCallback(_internal_error_text3)
5688 d.addCallback(lambda ignored:
5689 self.shouldHTTPError("GET errorboom_text",
5690 500, "Internal Server Error", None,
5691 self.GET, "ERRORBOOM"))
5692 def _internal_error_html4(body):
5693 self.failUnlessIn("<html>", body)
5694 d.addCallback(_internal_error_html4)
5696 def _flush_errors(res):
5697 # Trial: please ignore the CompletelyUnhandledError in the logs
5698 self.flushLoggedErrors(CompletelyUnhandledError)
5700 d.addBoth(_flush_errors)
5704 def test_blacklist(self):
5705 # download from a blacklisted URI, get an error
5706 self.basedir = "web/Grid/blacklist"
5708 c0 = self.g.clients[0]
5709 c0_basedir = c0.basedir
5710 fn = os.path.join(c0_basedir, "access.blacklist")
5712 DATA = "off-limits " * 50
5714 d = c0.upload(upload.Data(DATA, convergence=""))
5715 def _stash_uri_and_create_dir(ur):
5716 self.uri = ur.get_uri()
5717 self.url = "uri/"+self.uri
5718 u = uri.from_string_filenode(self.uri)
5719 self.si = u.get_storage_index()
5720 childnode = c0.create_node_from_uri(self.uri, None)
5721 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5722 d.addCallback(_stash_uri_and_create_dir)
5723 def _stash_dir(node):
5724 self.dir_node = node
5725 self.dir_uri = node.get_uri()
5726 self.dir_url = "uri/"+self.dir_uri
5727 d.addCallback(_stash_dir)
5728 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5729 def _check_dir_html(body):
5730 self.failUnlessIn("<html>", body)
5731 self.failUnlessIn("blacklisted.txt</a>", body)
5732 d.addCallback(_check_dir_html)
5733 d.addCallback(lambda ign: self.GET(self.url))
5734 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5736 def _blacklist(ign):
5738 f.write(" # this is a comment\n")
5740 f.write("\n") # also exercise blank lines
5741 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5743 # clients should be checking the blacklist each time, so we don't
5744 # need to restart the client
5745 d.addCallback(_blacklist)
5746 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5748 "Access Prohibited: off-limits",
5749 self.GET, self.url))
5751 # We should still be able to list the parent directory, in HTML...
5752 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5753 def _check_dir_html2(body):
5754 self.failUnlessIn("<html>", body)
5755 self.failUnlessIn("blacklisted.txt</strike>", body)
5756 d.addCallback(_check_dir_html2)
5758 # ... and in JSON (used by CLI).
5759 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5760 def _check_dir_json(res):
5761 data = simplejson.loads(res)
5762 self.failUnless(isinstance(data, list), data)
5763 self.failUnlessEqual(data[0], "dirnode")
5764 self.failUnless(isinstance(data[1], dict), data)
5765 self.failUnlessIn("children", data[1])
5766 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5767 childdata = data[1]["children"]["blacklisted.txt"]
5768 self.failUnless(isinstance(childdata, list), data)
5769 self.failUnlessEqual(childdata[0], "filenode")
5770 self.failUnless(isinstance(childdata[1], dict), data)
5771 d.addCallback(_check_dir_json)
5773 def _unblacklist(ign):
5774 open(fn, "w").close()
5775 # the Blacklist object watches mtime to tell when the file has
5776 # changed, but on windows this test will run faster than the
5777 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5778 # to force a reload.
5779 self.g.clients[0].blacklist.last_mtime -= 2.0
5780 d.addCallback(_unblacklist)
5782 # now a read should work
5783 d.addCallback(lambda ign: self.GET(self.url))
5784 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5786 # read again to exercise the blacklist-is-unchanged logic
5787 d.addCallback(lambda ign: self.GET(self.url))
5788 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5790 # now add a blacklisted directory, and make sure files under it are
5793 childnode = c0.create_node_from_uri(self.uri, None)
5794 return c0.create_dirnode({u"child": (childnode,{}) })
5795 d.addCallback(_add_dir)
5796 def _get_dircap(dn):
5797 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5798 self.dir_url_base = "uri/"+dn.get_write_uri()
5799 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5800 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5801 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5802 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5803 d.addCallback(_get_dircap)
5804 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5805 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5806 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5807 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5808 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5809 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5810 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5811 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5812 d.addCallback(lambda ign: self.GET(self.child_url))
5813 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5815 def _block_dir(ign):
5817 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5819 self.g.clients[0].blacklist.last_mtime -= 2.0
5820 d.addCallback(_block_dir)
5821 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5823 "Access Prohibited: dir-off-limits",
5824 self.GET, self.dir_url_base))
5825 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5827 "Access Prohibited: dir-off-limits",
5828 self.GET, self.dir_url_json1))
5829 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5831 "Access Prohibited: dir-off-limits",
5832 self.GET, self.dir_url_json2))
5833 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5835 "Access Prohibited: dir-off-limits",
5836 self.GET, self.dir_url_json_ro))
5837 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5839 "Access Prohibited: dir-off-limits",
5840 self.GET, self.child_url))
5844 class CompletelyUnhandledError(Exception):
5846 class ErrorBoom(rend.Page):
5847 def beforeRender(self, ctx):
5848 raise CompletelyUnhandledError("whoops")