1 import os.path, re, urllib, time, cgi
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow.util import escapeToXML
15 from nevow import rend
17 from allmydata import interfaces, uri, webish, dirnode
18 from allmydata.storage.shares import get_share_file
19 from allmydata.storage_client import StorageFarmBroker, StubServer
20 from allmydata.immutable import upload
21 from allmydata.immutable.downloader.status import DownloadStatus
22 from allmydata.dirnode import DirectoryNode
23 from allmydata.nodemaker import NodeMaker
24 from allmydata.unknown import UnknownNode
25 from allmydata.web import status, common
26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
27 from allmydata.util import fileutil, base32, hashutil
28 from allmydata.util.consumer import download_to_data
29 from allmydata.util.netstring import split_netstring
30 from allmydata.util.encodingutil import to_str
31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
32 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
33 make_mutable_file_uri, create_mutable_filenode
34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
35 from allmydata.mutable import servermap, publish, retrieve
36 import allmydata.test.common_util as testutil
37 from allmydata.test.no_network import GridTestMixin
38 from allmydata.test.common_web import HTTPClientGETFactory, \
40 from allmydata.client import Client, SecretHolder
41 from allmydata.introducer import IntroducerNode
43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
44 # create a webserver that works against them
46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
55 class FakeStatsProvider:
57 stats = {'stats': {}, 'counters': {}}
60 class FakeNodeMaker(NodeMaker):
65 'max_segment_size':128*1024 # 1024=KiB
67 def _create_lit(self, cap):
68 return FakeCHKFileNode(cap, self.all_contents)
69 def _create_immutable(self, cap):
70 return FakeCHKFileNode(cap, self.all_contents)
71 def _create_mutable(self, cap):
72 return FakeMutableFileNode(None, None,
73 self.encoding_params, None,
74 self.all_contents).init_from_cap(cap)
75 def create_mutable_file(self, contents="", keysize=None,
76 version=SDMF_VERSION):
77 n = FakeMutableFileNode(None, None, self.encoding_params, None,
79 return n.create(contents, version=version)
81 class FakeUploader(service.Service):
84 helper_connected = False
86 def upload(self, uploadable):
87 d = uploadable.get_size()
88 d.addCallback(lambda size: uploadable.read(size))
91 n = create_chk_filenode(data, self.all_contents)
92 ur = upload.UploadResults(file_size=len(data),
99 uri_extension_data={},
100 uri_extension_hash="fake",
101 verifycapstr="fakevcap")
102 ur.set_uri(n.get_uri())
104 d.addCallback(_got_data)
107 def get_helper_info(self):
108 return (self.helper_furl, self.helper_connected)
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeDisplayableServer(StubServer):
175 def __init__(self, serverid, nickname):
176 StubServer.__init__(self, serverid)
177 self.announcement = {"my-version": "allmydata-tahoe-fake",
178 "service-name": "storage",
179 "nickname": nickname}
180 def is_connected(self):
182 def get_permutation_seed(self):
184 def get_remote_host(self):
186 def get_last_loss_time(self):
188 def get_announcement_time(self):
190 def get_announcement(self):
191 return self.announcement
192 def get_nickname(self):
193 return self.announcement["nickname"]
195 class FakeBucketCounter(object):
197 return {"last-complete-bucket-count": 0}
198 def get_progress(self):
199 return {"estimated-time-per-cycle": 0,
200 "cycle-in-progress": False,
201 "remaining-wait-time": 0}
203 class FakeLeaseChecker(object):
205 self.expiration_enabled = False
207 self.override_lease_duration = None
208 self.sharetypes_to_expire = {}
210 return {"history": None}
211 def get_progress(self):
212 return {"estimated-time-per-cycle": 0,
213 "cycle-in-progress": False,
214 "remaining-wait-time": 0}
216 class FakeStorageServer(service.MultiService):
218 def __init__(self, nodeid, nickname):
219 service.MultiService.__init__(self)
220 self.my_nodeid = nodeid
221 self.nickname = nickname
222 self.bucket_counter = FakeBucketCounter()
223 self.lease_checker = FakeLeaseChecker()
225 return {"storage_server.accepting_immutable_shares": False}
227 class FakeClient(Client):
229 # don't upcall to Client.__init__, since we only want to initialize a
231 service.MultiService.__init__(self)
232 self.all_contents = {}
233 self.nodeid = "fake_nodeid"
234 self.nickname = u"fake_nickname \u263A"
235 self.introducer_furl = "None"
236 self.stats_provider = FakeStatsProvider()
237 self._secret_holder = SecretHolder("lease secret", "convergence secret")
239 self.convergence = "some random string"
240 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
241 # fake knowledge of another server
242 self.storage_broker.test_add_server("other_nodeid",
243 FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
244 self.introducer_client = None
245 self.history = FakeHistory()
246 self.uploader = FakeUploader()
247 self.uploader.all_contents = self.all_contents
248 self.uploader.setServiceParent(self)
249 self.blacklist = None
250 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
253 self.nodemaker.all_contents = self.all_contents
254 self.mutable_file_default = SDMF_VERSION
255 self.addService(FakeStorageServer(self.nodeid, self.nickname))
257 def startService(self):
258 return service.MultiService.startService(self)
259 def stopService(self):
260 return service.MultiService.stopService(self)
262 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
264 class WebMixin(object):
266 self.s = FakeClient()
267 self.s.startService()
268 self.staticdir = self.mktemp()
270 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
272 self.ws.setServiceParent(self.s)
273 self.webish_port = self.ws.getPortnum()
274 self.webish_url = self.ws.getURL()
275 assert self.webish_url.endswith("/")
276 self.webish_url = self.webish_url[:-1] # these tests add their own /
278 l = [ self.s.create_dirnode() for x in range(6) ]
279 d = defer.DeferredList(l)
281 self.public_root = res[0][1]
282 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
283 self.public_url = "/uri/" + self.public_root.get_uri()
284 self.private_root = res[1][1]
288 self._foo_uri = foo.get_uri()
289 self._foo_readonly_uri = foo.get_readonly_uri()
290 self._foo_verifycap = foo.get_verify_cap().to_string()
291 # NOTE: we ignore the deferred on all set_uri() calls, because we
292 # know the fake nodes do these synchronously
293 self.public_root.set_uri(u"foo", foo.get_uri(),
294 foo.get_readonly_uri())
296 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
297 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
298 self._bar_txt_verifycap = n.get_verify_cap().to_string()
301 # XXX: Do we ever use this?
302 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
304 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
307 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
308 assert self._quux_txt_uri.startswith("URI:MDMF")
309 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
311 foo.set_uri(u"empty", res[3][1].get_uri(),
312 res[3][1].get_readonly_uri())
313 sub_uri = res[4][1].get_uri()
314 self._sub_uri = sub_uri
315 foo.set_uri(u"sub", sub_uri, sub_uri)
316 sub = self.s.create_node_from_uri(sub_uri)
319 _ign, n, blocking_uri = self.makefile(1)
320 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
322 # filenode to test for html encoding issues
323 self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
324 self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
325 self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
326 self._htmlname_escaped = escapeToXML(self._htmlname_raw)
327 self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
328 self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
329 self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
330 foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
332 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
333 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
334 # still think of it as an umlaut
335 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
337 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
338 self._baz_file_uri = baz_file
339 sub.set_uri(u"baz.txt", baz_file, baz_file)
341 _ign, n, self._bad_file_uri = self.makefile(3)
342 # this uri should not be downloadable
343 del self.s.all_contents[self._bad_file_uri]
346 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
347 rodir.get_readonly_uri())
348 rodir.set_uri(u"nor", baz_file, baz_file)
354 # public/foo/quux.txt
355 # public/foo/blockingfile
356 # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
359 # public/foo/sub/baz.txt
361 # public/reedownlee/nor
362 self.NEWFILE_CONTENTS = "newfile contents\n"
364 return foo.get_metadata_for(u"bar.txt")
366 def _got_metadata(metadata):
367 self._bar_txt_metadata = metadata
368 d.addCallback(_got_metadata)
371 def get_all_contents(self):
372 return self.s.all_contents
374 def makefile(self, number):
375 contents = "contents of file %s\n" % number
376 n = create_chk_filenode(contents, self.get_all_contents())
377 return contents, n, n.get_uri()
379 def makefile_mutable(self, number, mdmf=False):
380 contents = "contents of mutable file %s\n" % number
381 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
382 return contents, n, n.get_uri(), n.get_readonly_uri()
385 return self.s.stopService()
387 def failUnlessIsBarDotTxt(self, res):
388 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
390 def failUnlessIsQuuxDotTxt(self, res):
391 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
393 def failUnlessIsBazDotTxt(self, res):
394 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
396 def failUnlessIsSubBazDotTxt(self, res):
397 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
399 def failUnlessIsBarJSON(self, res):
400 data = simplejson.loads(res)
401 self.failUnless(isinstance(data, list))
402 self.failUnlessEqual(data[0], "filenode")
403 self.failUnless(isinstance(data[1], dict))
404 self.failIf(data[1]["mutable"])
405 self.failIfIn("rw_uri", data[1]) # immutable
406 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
407 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
408 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
410 def failUnlessIsQuuxJSON(self, res, readonly=False):
411 data = simplejson.loads(res)
412 self.failUnless(isinstance(data, list))
413 self.failUnlessEqual(data[0], "filenode")
414 self.failUnless(isinstance(data[1], dict))
416 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
418 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
419 self.failUnless(metadata['mutable'])
421 self.failIfIn("rw_uri", metadata)
423 self.failUnlessIn("rw_uri", metadata)
424 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
425 self.failUnlessIn("ro_uri", metadata)
426 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
427 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
429 def failUnlessIsFooJSON(self, res):
430 data = simplejson.loads(res)
431 self.failUnless(isinstance(data, list))
432 self.failUnlessEqual(data[0], "dirnode", res)
433 self.failUnless(isinstance(data[1], dict))
434 self.failUnless(data[1]["mutable"])
435 self.failUnlessIn("rw_uri", data[1]) # mutable
436 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
437 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
438 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
440 kidnames = sorted([unicode(n) for n in data[1]["children"]])
441 self.failUnlessEqual(kidnames,
442 [self._htmlname_unicode, u"bar.txt", u"baz.txt",
443 u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
444 kids = dict( [(unicode(name),value)
446 in data[1]["children"].iteritems()] )
447 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
448 self.failUnlessIn("metadata", kids[u"sub"][1])
449 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
450 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
451 self.failUnlessIn("linkcrtime", tahoe_md)
452 self.failUnlessIn("linkmotime", tahoe_md)
453 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
454 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
455 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
456 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
457 self._bar_txt_verifycap)
458 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
459 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
460 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
461 self._bar_txt_metadata["tahoe"]["linkcrtime"])
462 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
464 self.failUnlessIn("quux.txt", kids)
465 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
467 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
468 self._quux_txt_readonly_uri)
470 def GET(self, urlpath, followRedirect=False, return_response=False,
472 # if return_response=True, this fires with (data, statuscode,
473 # respheaders) instead of just data.
474 assert not isinstance(urlpath, unicode)
475 url = self.webish_url + urlpath
476 factory = HTTPClientGETFactory(url, method="GET",
477 followRedirect=followRedirect, **kwargs)
478 reactor.connectTCP("localhost", self.webish_port, factory)
481 return (data, factory.status, factory.response_headers)
483 d.addCallback(_got_data)
484 return factory.deferred
486 def HEAD(self, urlpath, return_response=False, **kwargs):
487 # this requires some surgery, because twisted.web.client doesn't want
488 # to give us back the response headers.
489 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
490 reactor.connectTCP("localhost", self.webish_port, factory)
493 return (data, factory.status, factory.response_headers)
495 d.addCallback(_got_data)
496 return factory.deferred
498 def PUT(self, urlpath, data, **kwargs):
499 url = self.webish_url + urlpath
500 return client.getPage(url, method="PUT", postdata=data, **kwargs)
502 def DELETE(self, urlpath):
503 url = self.webish_url + urlpath
504 return client.getPage(url, method="DELETE")
506 def POST(self, urlpath, followRedirect=False, **fields):
507 sepbase = "boogabooga"
511 form.append('Content-Disposition: form-data; name="_charset"')
515 for name, value in fields.iteritems():
516 if isinstance(value, tuple):
517 filename, value = value
518 form.append('Content-Disposition: form-data; name="%s"; '
519 'filename="%s"' % (name, filename.encode("utf-8")))
521 form.append('Content-Disposition: form-data; name="%s"' % name)
523 if isinstance(value, unicode):
524 value = value.encode("utf-8")
527 assert isinstance(value, str)
534 body = "\r\n".join(form) + "\r\n"
535 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
536 return self.POST2(urlpath, body, headers, followRedirect)
538 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
539 url = self.webish_url + urlpath
540 return client.getPage(url, method="POST", postdata=body,
541 headers=headers, followRedirect=followRedirect)
543 def shouldFail(self, res, expected_failure, which,
544 substring=None, response_substring=None):
545 if isinstance(res, failure.Failure):
546 res.trap(expected_failure)
548 self.failUnlessIn(substring, str(res), which)
549 if response_substring:
550 self.failUnlessIn(response_substring, res.value.response, which)
552 self.fail("%s was supposed to raise %s, not get '%s'" %
553 (which, expected_failure, res))
555 def shouldFail2(self, expected_failure, which, substring,
557 callable, *args, **kwargs):
558 assert substring is None or isinstance(substring, str)
559 assert response_substring is None or isinstance(response_substring, str)
560 d = defer.maybeDeferred(callable, *args, **kwargs)
562 if isinstance(res, failure.Failure):
563 res.trap(expected_failure)
565 self.failUnlessIn(substring, str(res),
566 "'%s' not in '%s' (response is '%s') for test '%s'" % \
567 (substring, str(res),
568 getattr(res.value, "response", ""),
570 if response_substring:
571 self.failUnlessIn(response_substring, res.value.response,
572 "'%s' not in '%s' for test '%s'" % \
573 (response_substring, res.value.response,
576 self.fail("%s was supposed to raise %s, not get '%s'" %
577 (which, expected_failure, res))
581 def should404(self, res, which):
582 if isinstance(res, failure.Failure):
583 res.trap(error.Error)
584 self.failUnlessReallyEqual(res.value.status, "404")
586 self.fail("%s was supposed to Error(404), not get '%s'" %
589 def should302(self, res, which):
590 if isinstance(res, failure.Failure):
591 res.trap(error.Error)
592 self.failUnlessReallyEqual(res.value.status, "302")
594 self.fail("%s was supposed to Error(302), not get '%s'" %
598 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
599 def test_create(self):
602 def test_welcome(self):
605 self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
606 self.failUnlessIn(FAVICON_MARKUP, res)
607 self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
608 self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
609 self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
610 res_u = res.decode('utf-8')
611 self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
612 self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
613 self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
615 self.s.basedir = 'web/test_welcome'
616 fileutil.make_dirs("web/test_welcome")
617 fileutil.make_dirs("web/test_welcome/private")
619 d.addCallback(_check)
622 def test_introducer_status(self):
623 class MockIntroducerClient(object):
624 def __init__(self, connected):
625 self.connected = connected
626 def connected_to_introducer(self):
627 return self.connected
629 d = defer.succeed(None)
631 # introducer not connected, unguessable furl
632 def _set_introducer_not_connected_unguessable(ign):
633 self.s.introducer_furl = "pb://someIntroducer/secret"
634 self.s.introducer_client = MockIntroducerClient(False)
636 d.addCallback(_set_introducer_not_connected_unguessable)
637 def _check_introducer_not_connected_unguessable(res):
638 html = res.replace('\n', ' ')
639 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
640 self.failIfIn('pb://someIntroducer/secret', html)
641 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
642 d.addCallback(_check_introducer_not_connected_unguessable)
644 # introducer connected, unguessable furl
645 def _set_introducer_connected_unguessable(ign):
646 self.s.introducer_furl = "pb://someIntroducer/secret"
647 self.s.introducer_client = MockIntroducerClient(True)
649 d.addCallback(_set_introducer_connected_unguessable)
650 def _check_introducer_connected_unguessable(res):
651 html = res.replace('\n', ' ')
652 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
653 self.failIfIn('pb://someIntroducer/secret', html)
654 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
655 d.addCallback(_check_introducer_connected_unguessable)
657 # introducer connected, guessable furl
658 def _set_introducer_connected_guessable(ign):
659 self.s.introducer_furl = "pb://someIntroducer/introducer"
660 self.s.introducer_client = MockIntroducerClient(True)
662 d.addCallback(_set_introducer_connected_guessable)
663 def _check_introducer_connected_guessable(res):
664 html = res.replace('\n', ' ')
665 self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
666 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
667 d.addCallback(_check_introducer_connected_guessable)
670 def test_helper_status(self):
671 d = defer.succeed(None)
673 # set helper furl to None
674 def _set_no_helper(ign):
675 self.s.uploader.helper_furl = None
677 d.addCallback(_set_no_helper)
678 def _check_no_helper(res):
679 html = res.replace('\n', ' ')
680 self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
681 d.addCallback(_check_no_helper)
683 # enable helper, not connected
684 def _set_helper_not_connected(ign):
685 self.s.uploader.helper_furl = "pb://someHelper/secret"
686 self.s.uploader.helper_connected = False
688 d.addCallback(_set_helper_not_connected)
689 def _check_helper_not_connected(res):
690 html = res.replace('\n', ' ')
691 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
692 self.failIfIn('pb://someHelper/secret', html)
693 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
694 d.addCallback(_check_helper_not_connected)
696 # enable helper, connected
697 def _set_helper_connected(ign):
698 self.s.uploader.helper_furl = "pb://someHelper/secret"
699 self.s.uploader.helper_connected = True
701 d.addCallback(_set_helper_connected)
702 def _check_helper_connected(res):
703 html = res.replace('\n', ' ')
704 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
705 self.failIfIn('pb://someHelper/secret', html)
706 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
707 d.addCallback(_check_helper_connected)
710 def test_storage(self):
711 d = self.GET("/storage")
713 self.failUnlessIn('Storage Server Status', res)
714 self.failUnlessIn(FAVICON_MARKUP, res)
715 res_u = res.decode('utf-8')
716 self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
717 d.addCallback(_check)
720 def test_status(self):
721 h = self.s.get_history()
722 dl_num = h.list_all_download_statuses()[0].get_counter()
723 ul_num = h.list_all_upload_statuses()[0].get_counter()
724 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
725 pub_num = h.list_all_publish_statuses()[0].get_counter()
726 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
727 d = self.GET("/status", followRedirect=True)
729 self.failUnlessIn('Recent and Active Operations', res)
730 self.failUnlessIn('"down-%d"' % dl_num, res)
731 self.failUnlessIn('"up-%d"' % ul_num, res)
732 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
733 self.failUnlessIn('"publish-%d"' % pub_num, res)
734 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
735 d.addCallback(_check)
736 d.addCallback(lambda res: self.GET("/status/?t=json"))
737 def _check_json(res):
738 data = simplejson.loads(res)
739 self.failUnless(isinstance(data, dict))
740 #active = data["active"]
741 # TODO: test more. We need a way to fake an active operation
743 d.addCallback(_check_json)
745 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
747 self.failUnlessIn("File Download Status", res)
748 d.addCallback(_check_dl)
749 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
750 def _check_dl_json(res):
751 data = simplejson.loads(res)
752 self.failUnless(isinstance(data, dict))
753 self.failUnlessIn("read", data)
754 self.failUnlessEqual(data["read"][0]["length"], 120)
755 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
756 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
757 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
758 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
759 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
760 # serverids[] keys are strings, since that's what JSON does, but
761 # we'd really like them to be ints
762 self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
763 self.failUnless(data["serverids"].has_key("1"),
764 str(data["serverids"]))
765 self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
766 str(data["serverids"]))
767 self.failUnlessEqual(data["server_info"][phwr_id]["short"],
769 self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
771 self.failUnlessIn("dyhb", data)
772 self.failUnlessIn("misc", data)
773 d.addCallback(_check_dl_json)
774 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
776 self.failUnlessIn("File Upload Status", res)
777 d.addCallback(_check_ul)
778 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
779 def _check_mapupdate(res):
780 self.failUnlessIn("Mutable File Servermap Update Status", res)
781 d.addCallback(_check_mapupdate)
782 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
783 def _check_publish(res):
784 self.failUnlessIn("Mutable File Publish Status", res)
785 d.addCallback(_check_publish)
786 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
787 def _check_retrieve(res):
788 self.failUnlessIn("Mutable File Retrieve Status", res)
789 d.addCallback(_check_retrieve)
793 def test_status_numbers(self):
794 drrm = status.DownloadResultsRendererMixin()
795 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
796 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
797 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
798 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
799 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
800 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
801 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
802 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
803 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
805 urrm = status.UploadResultsRendererMixin()
806 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
807 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
808 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
809 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
810 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
811 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
812 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
813 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
814 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
816 def test_GET_FILEURL(self):
817 d = self.GET(self.public_url + "/foo/bar.txt")
818 d.addCallback(self.failUnlessIsBarDotTxt)
821 def test_GET_FILEURL_range(self):
822 headers = {"range": "bytes=1-10"}
823 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
824 return_response=True)
825 def _got((res, status, headers)):
826 self.failUnlessReallyEqual(int(status), 206)
827 self.failUnless(headers.has_key("content-range"))
828 self.failUnlessReallyEqual(headers["content-range"][0],
829 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
830 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
834 def test_GET_FILEURL_partial_range(self):
835 headers = {"range": "bytes=5-"}
836 length = len(self.BAR_CONTENTS)
837 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
838 return_response=True)
839 def _got((res, status, headers)):
840 self.failUnlessReallyEqual(int(status), 206)
841 self.failUnless(headers.has_key("content-range"))
842 self.failUnlessReallyEqual(headers["content-range"][0],
843 "bytes 5-%d/%d" % (length-1, length))
844 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
848 def test_GET_FILEURL_partial_end_range(self):
849 headers = {"range": "bytes=-5"}
850 length = len(self.BAR_CONTENTS)
851 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
852 return_response=True)
853 def _got((res, status, headers)):
854 self.failUnlessReallyEqual(int(status), 206)
855 self.failUnless(headers.has_key("content-range"))
856 self.failUnlessReallyEqual(headers["content-range"][0],
857 "bytes %d-%d/%d" % (length-5, length-1, length))
858 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
862 def test_GET_FILEURL_partial_range_overrun(self):
863 headers = {"range": "bytes=100-200"}
864 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
865 "416 Requested Range not satisfiable",
866 "First beyond end of file",
867 self.GET, self.public_url + "/foo/bar.txt",
871 def test_HEAD_FILEURL_range(self):
872 headers = {"range": "bytes=1-10"}
873 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
874 return_response=True)
875 def _got((res, status, headers)):
876 self.failUnlessReallyEqual(res, "")
877 self.failUnlessReallyEqual(int(status), 206)
878 self.failUnless(headers.has_key("content-range"))
879 self.failUnlessReallyEqual(headers["content-range"][0],
880 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
884 def test_HEAD_FILEURL_partial_range(self):
885 headers = {"range": "bytes=5-"}
886 length = len(self.BAR_CONTENTS)
887 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
888 return_response=True)
889 def _got((res, status, headers)):
890 self.failUnlessReallyEqual(int(status), 206)
891 self.failUnless(headers.has_key("content-range"))
892 self.failUnlessReallyEqual(headers["content-range"][0],
893 "bytes 5-%d/%d" % (length-1, length))
897 def test_HEAD_FILEURL_partial_end_range(self):
898 headers = {"range": "bytes=-5"}
899 length = len(self.BAR_CONTENTS)
900 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
901 return_response=True)
902 def _got((res, status, headers)):
903 self.failUnlessReallyEqual(int(status), 206)
904 self.failUnless(headers.has_key("content-range"))
905 self.failUnlessReallyEqual(headers["content-range"][0],
906 "bytes %d-%d/%d" % (length-5, length-1, length))
910 def test_HEAD_FILEURL_partial_range_overrun(self):
911 headers = {"range": "bytes=100-200"}
912 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
913 "416 Requested Range not satisfiable",
915 self.HEAD, self.public_url + "/foo/bar.txt",
919 def test_GET_FILEURL_range_bad(self):
920 headers = {"range": "BOGUS=fizbop-quarnak"}
921 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
922 return_response=True)
923 def _got((res, status, headers)):
924 self.failUnlessReallyEqual(int(status), 200)
925 self.failUnless(not headers.has_key("content-range"))
926 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
930 def test_HEAD_FILEURL(self):
931 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
932 def _got((res, status, headers)):
933 self.failUnlessReallyEqual(res, "")
934 self.failUnlessReallyEqual(headers["content-length"][0],
935 str(len(self.BAR_CONTENTS)))
936 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
940 def test_GET_FILEURL_named(self):
941 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
942 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
943 d = self.GET(base + "/@@name=/blah.txt")
944 d.addCallback(self.failUnlessIsBarDotTxt)
945 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
946 d.addCallback(self.failUnlessIsBarDotTxt)
947 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
948 d.addCallback(self.failUnlessIsBarDotTxt)
949 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
950 d.addCallback(self.failUnlessIsBarDotTxt)
951 save_url = base + "?save=true&filename=blah.txt"
952 d.addCallback(lambda res: self.GET(save_url))
953 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
954 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
955 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
956 u_url = base + "?save=true&filename=" + u_fn_e
957 d.addCallback(lambda res: self.GET(u_url))
958 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
961 def test_PUT_FILEURL_named_bad(self):
962 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
963 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
965 "/file can only be used with GET or HEAD",
966 self.PUT, base + "/@@name=/blah.txt", "")
970 def test_GET_DIRURL_named_bad(self):
971 base = "/file/%s" % urllib.quote(self._foo_uri)
972 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
975 self.GET, base + "/@@name=/blah.txt")
978 def test_GET_slash_file_bad(self):
979 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
981 "/file must be followed by a file-cap and a name",
985 def test_GET_unhandled_URI_named(self):
986 contents, n, newuri = self.makefile(12)
987 verifier_cap = n.get_verify_cap().to_string()
988 base = "/file/%s" % urllib.quote(verifier_cap)
989 # client.create_node_from_uri() can't handle verify-caps
990 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
991 "400 Bad Request", "is not a file-cap",
995 def test_GET_unhandled_URI(self):
996 contents, n, newuri = self.makefile(12)
997 verifier_cap = n.get_verify_cap().to_string()
998 base = "/uri/%s" % urllib.quote(verifier_cap)
999 # client.create_node_from_uri() can't handle verify-caps
1000 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1002 "GET unknown URI type: can only do t=info",
1006 def test_GET_FILE_URI(self):
1007 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1009 d.addCallback(self.failUnlessIsBarDotTxt)
1012 def test_GET_FILE_URI_mdmf(self):
1013 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1015 d.addCallback(self.failUnlessIsQuuxDotTxt)
1018 def test_GET_FILE_URI_mdmf_extensions(self):
1019 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1021 d.addCallback(self.failUnlessIsQuuxDotTxt)
1024 def test_GET_FILE_URI_mdmf_readonly(self):
1025 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1027 d.addCallback(self.failUnlessIsQuuxDotTxt)
1030 def test_GET_FILE_URI_badchild(self):
1031 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1032 errmsg = "Files have no children, certainly not named 'boguschild'"
1033 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1034 "400 Bad Request", errmsg,
1038 def test_PUT_FILE_URI_badchild(self):
1039 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1040 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1041 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1042 "400 Bad Request", errmsg,
1046 def test_PUT_FILE_URI_mdmf(self):
1047 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1048 self._quux_new_contents = "new_contents"
1050 d.addCallback(lambda res:
1051 self.failUnlessIsQuuxDotTxt(res))
1052 d.addCallback(lambda ignored:
1053 self.PUT(base, self._quux_new_contents))
1054 d.addCallback(lambda ignored:
1056 d.addCallback(lambda res:
1057 self.failUnlessReallyEqual(res, self._quux_new_contents))
1060 def test_PUT_FILE_URI_mdmf_extensions(self):
1061 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1062 self._quux_new_contents = "new_contents"
1064 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1065 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1066 d.addCallback(lambda ignored: self.GET(base))
1067 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1071 def test_PUT_FILE_URI_mdmf_readonly(self):
1072 # We're not allowed to PUT things to a readonly cap.
1073 base = "/uri/%s" % self._quux_txt_readonly_uri
1075 d.addCallback(lambda res:
1076 self.failUnlessIsQuuxDotTxt(res))
1077 # What should we get here? We get a 500 error now; that's not right.
1078 d.addCallback(lambda ignored:
1079 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1080 "400 Bad Request", "read-only cap",
1081 self.PUT, base, "new data"))
1084 def test_PUT_FILE_URI_sdmf_readonly(self):
1085 # We're not allowed to put things to a readonly cap.
1086 base = "/uri/%s" % self._baz_txt_readonly_uri
1088 d.addCallback(lambda res:
1089 self.failUnlessIsBazDotTxt(res))
1090 d.addCallback(lambda ignored:
1091 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1092 "400 Bad Request", "read-only cap",
1093 self.PUT, base, "new_data"))
1096 def test_GET_etags(self):
1098 def _check_etags(uri):
1100 d2 = _get_etag(uri, 'json')
1101 d = defer.DeferredList([d1, d2], consumeErrors=True)
1102 def _check(results):
1103 # All deferred must succeed
1104 self.failUnless(all([r[0] for r in results]))
1105 # the etag for the t=json form should be just like the etag
1106 # fo the default t='' form, but with a 'json' suffix
1107 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1108 d.addCallback(_check)
1111 def _get_etag(uri, t=''):
1112 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1113 d = self.GET(targetbase, return_response=True, followRedirect=True)
1114 def _just_the_etag(result):
1115 data, response, headers = result
1116 etag = headers['etag'][0]
1117 if uri.startswith('URI:DIR'):
1118 self.failUnless(etag.startswith('DIR:'), etag)
1120 return d.addCallback(_just_the_etag)
1122 # Check that etags work with immutable directories
1123 (newkids, caps) = self._create_immutable_children()
1124 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1125 simplejson.dumps(newkids))
1126 def _stash_immdir_uri(uri):
1127 self._immdir_uri = uri
1129 d.addCallback(_stash_immdir_uri)
1130 d.addCallback(_check_etags)
1132 # Check that etags work with immutable files
1133 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1135 # use the ETag on GET
1136 def _check_match(ign):
1137 uri = "/uri/%s" % self._bar_txt_uri
1138 d = self.GET(uri, return_response=True)
1140 d.addCallback(lambda (data, code, headers):
1142 # do a GET that's supposed to match the ETag
1143 d.addCallback(lambda etag:
1144 self.GET(uri, return_response=True,
1145 headers={"If-None-Match": etag}))
1146 # make sure it short-circuited (304 instead of 200)
1147 d.addCallback(lambda (data, code, headers):
1148 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1150 d.addCallback(_check_match)
1152 def _no_etag(uri, t):
1153 target = "/uri/%s?t=%s" % (uri, t)
1154 d = self.GET(target, return_response=True, followRedirect=True)
1155 d.addCallback(lambda (data, code, headers):
1156 self.failIf("etag" in headers, target))
1158 def _yes_etag(uri, t):
1159 target = "/uri/%s?t=%s" % (uri, t)
1160 d = self.GET(target, return_response=True, followRedirect=True)
1161 d.addCallback(lambda (data, code, headers):
1162 self.failUnless("etag" in headers, target))
1165 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1166 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1167 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1168 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1169 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1171 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1172 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1173 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1174 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1175 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1176 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1180 # TODO: version of this with a Unicode filename
1181 def test_GET_FILEURL_save(self):
1182 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1183 return_response=True)
1184 def _got((res, statuscode, headers)):
1185 content_disposition = headers["content-disposition"][0]
1186 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1187 self.failUnlessIsBarDotTxt(res)
1191 def test_GET_FILEURL_missing(self):
1192 d = self.GET(self.public_url + "/foo/missing")
1193 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1196 def test_GET_FILEURL_info_mdmf(self):
1197 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1199 self.failUnlessIn("mutable file (mdmf)", res)
1200 self.failUnlessIn(self._quux_txt_uri, res)
1201 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1205 def test_GET_FILEURL_info_mdmf_readonly(self):
1206 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1208 self.failUnlessIn("mutable file (mdmf)", res)
1209 self.failIfIn(self._quux_txt_uri, res)
1210 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1214 def test_GET_FILEURL_info_sdmf(self):
1215 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1217 self.failUnlessIn("mutable file (sdmf)", res)
1218 self.failUnlessIn(self._baz_txt_uri, res)
1222 def test_GET_FILEURL_info_mdmf_extensions(self):
1223 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1225 self.failUnlessIn("mutable file (mdmf)", res)
1226 self.failUnlessIn(self._quux_txt_uri, res)
1227 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1231 def test_PUT_overwrite_only_files(self):
1232 # create a directory, put a file in that directory.
1233 contents, n, filecap = self.makefile(8)
1234 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1235 d.addCallback(lambda res:
1236 self.PUT(self.public_url + "/foo/dir/file1.txt",
1237 self.NEWFILE_CONTENTS))
1238 # try to overwrite the file with replace=only-files
1239 # (this should work)
1240 d.addCallback(lambda res:
1241 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1243 d.addCallback(lambda res:
1244 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1245 "There was already a child by that name, and you asked me "
1246 "to not replace it",
1247 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1251 def test_PUT_NEWFILEURL(self):
1252 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1253 # TODO: we lose the response code, so we can't check this
1254 #self.failUnlessReallyEqual(responsecode, 201)
1255 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1256 d.addCallback(lambda res:
1257 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1258 self.NEWFILE_CONTENTS))
1261 def test_PUT_NEWFILEURL_not_mutable(self):
1262 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1263 self.NEWFILE_CONTENTS)
1264 # TODO: we lose the response code, so we can't check this
1265 #self.failUnlessReallyEqual(responsecode, 201)
1266 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1267 d.addCallback(lambda res:
1268 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1269 self.NEWFILE_CONTENTS))
1272 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1273 # this should get us a few segments of an MDMF mutable file,
1274 # which we can then test for.
1275 contents = self.NEWFILE_CONTENTS * 300000
1276 d = self.PUT("/uri?format=mdmf",
1278 def _got_filecap(filecap):
1279 self.failUnless(filecap.startswith("URI:MDMF"))
1281 d.addCallback(_got_filecap)
1282 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1283 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1286 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1287 contents = self.NEWFILE_CONTENTS * 300000
1288 d = self.PUT("/uri?format=sdmf",
1290 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1291 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1294 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1295 contents = self.NEWFILE_CONTENTS * 300000
1296 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1297 400, "Bad Request", "Unknown format: foo",
1298 self.PUT, "/uri?format=foo",
1301 def test_PUT_NEWFILEURL_range_bad(self):
1302 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1303 target = self.public_url + "/foo/new.txt"
1304 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1305 "501 Not Implemented",
1306 "Content-Range in PUT not yet supported",
1307 # (and certainly not for immutable files)
1308 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1310 d.addCallback(lambda res:
1311 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1314 def test_PUT_NEWFILEURL_mutable(self):
1315 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1316 self.NEWFILE_CONTENTS)
1317 # TODO: we lose the response code, so we can't check this
1318 #self.failUnlessReallyEqual(responsecode, 201)
1319 def _check_uri(res):
1320 u = uri.from_string_mutable_filenode(res)
1321 self.failUnless(u.is_mutable())
1322 self.failIf(u.is_readonly())
1324 d.addCallback(_check_uri)
1325 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1326 d.addCallback(lambda res:
1327 self.failUnlessMutableChildContentsAre(self._foo_node,
1329 self.NEWFILE_CONTENTS))
1332 def test_PUT_NEWFILEURL_mutable_toobig(self):
1333 # It is okay to upload large mutable files, so we should be able
1335 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1336 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1339 def test_PUT_NEWFILEURL_replace(self):
1340 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1341 # TODO: we lose the response code, so we can't check this
1342 #self.failUnlessReallyEqual(responsecode, 200)
1343 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1344 d.addCallback(lambda res:
1345 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1346 self.NEWFILE_CONTENTS))
1349 def test_PUT_NEWFILEURL_bad_t(self):
1350 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1351 "PUT to a file: bad t=bogus",
1352 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1356 def test_PUT_NEWFILEURL_no_replace(self):
1357 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1358 self.NEWFILE_CONTENTS)
1359 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1361 "There was already a child by that name, and you asked me "
1362 "to not replace it")
1365 def test_PUT_NEWFILEURL_mkdirs(self):
1366 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1368 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1369 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1370 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1371 d.addCallback(lambda res:
1372 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1373 self.NEWFILE_CONTENTS))
1376 def test_PUT_NEWFILEURL_blocked(self):
1377 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1378 self.NEWFILE_CONTENTS)
1379 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1381 "Unable to create directory 'blockingfile': a file was in the way")
1384 def test_PUT_NEWFILEURL_emptyname(self):
1385 # an empty pathname component (i.e. a double-slash) is disallowed
1386 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1388 "The webapi does not allow empty pathname components",
1389 self.PUT, self.public_url + "/foo//new.txt", "")
1392 def test_DELETE_FILEURL(self):
1393 d = self.DELETE(self.public_url + "/foo/bar.txt")
1394 d.addCallback(lambda res:
1395 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1398 def test_DELETE_FILEURL_missing(self):
1399 d = self.DELETE(self.public_url + "/foo/missing")
1400 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1403 def test_DELETE_FILEURL_missing2(self):
1404 d = self.DELETE(self.public_url + "/missing/missing")
1405 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1408 def failUnlessHasBarDotTxtMetadata(self, res):
1409 data = simplejson.loads(res)
1410 self.failUnless(isinstance(data, list))
1411 self.failUnlessIn("metadata", data[1])
1412 self.failUnlessIn("tahoe", data[1]["metadata"])
1413 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1414 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1415 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1416 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1418 def test_GET_FILEURL_json(self):
1419 # twisted.web.http.parse_qs ignores any query args without an '=', so
1420 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1421 # instead. This may make it tricky to emulate the S3 interface
1423 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1425 self.failUnlessIsBarJSON(data)
1426 self.failUnlessHasBarDotTxtMetadata(data)
1428 d.addCallback(_check1)
1431 def test_GET_FILEURL_json_mutable_type(self):
1432 # The JSON should include format, which says whether the
1433 # file is SDMF or MDMF
1434 d = self.PUT("/uri?format=mdmf",
1435 self.NEWFILE_CONTENTS * 300000)
1436 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1437 def _got_json(json, version):
1438 data = simplejson.loads(json)
1439 assert "filenode" == data[0]
1441 assert isinstance(data, dict)
1443 self.failUnlessIn("format", data)
1444 self.failUnlessEqual(data["format"], version)
1446 d.addCallback(_got_json, "MDMF")
1447 # Now make an SDMF file and check that it is reported correctly.
1448 d.addCallback(lambda ignored:
1449 self.PUT("/uri?format=sdmf",
1450 self.NEWFILE_CONTENTS * 300000))
1451 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1452 d.addCallback(_got_json, "SDMF")
1455 def test_GET_FILEURL_json_mdmf(self):
1456 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1457 d.addCallback(self.failUnlessIsQuuxJSON)
1460 def test_GET_FILEURL_json_missing(self):
1461 d = self.GET(self.public_url + "/foo/missing?json")
1462 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1465 def test_GET_FILEURL_uri(self):
1466 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1468 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1469 d.addCallback(_check)
1470 d.addCallback(lambda res:
1471 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1473 # for now, for files, uris and readonly-uris are the same
1474 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1475 d.addCallback(_check2)
1478 def test_GET_FILEURL_badtype(self):
1479 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1482 self.public_url + "/foo/bar.txt?t=bogus")
1485 def test_CSS_FILE(self):
1486 d = self.GET("/tahoe.css", followRedirect=True)
1488 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1489 self.failUnless(CSS_STYLE.search(res), res)
1490 d.addCallback(_check)
1493 def test_GET_FILEURL_uri_missing(self):
1494 d = self.GET(self.public_url + "/foo/missing?t=uri")
1495 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1498 def _check_upload_and_mkdir_forms(self, html):
1499 # We should have a form to create a file, with radio buttons that allow
1500 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1501 self.failUnlessIn('name="t" value="upload"', html)
1502 self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1503 self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1504 self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1506 # We should also have the ability to create a mutable directory, with
1507 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1508 # or MDMF directory.
1509 self.failUnlessIn('name="t" value="mkdir"', html)
1510 self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1511 self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1513 self.failUnlessIn(FAVICON_MARKUP, html)
1515 def test_GET_DIRECTORY_html(self):
1516 d = self.GET(self.public_url + "/foo", followRedirect=True)
1518 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1519 self._check_upload_and_mkdir_forms(html)
1520 self.failUnlessIn("quux", html)
1521 d.addCallback(_check)
1524 def test_GET_DIRECTORY_html_filenode_encoding(self):
1525 d = self.GET(self.public_url + "/foo", followRedirect=True)
1527 # Check if encoded entries are there
1528 self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1529 + self._htmlname_escaped + '</a>', html)
1530 self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1531 self.failIfIn(self._htmlname_escaped_double, html)
1532 # Make sure that Nevow escaping actually works by checking for unsafe characters
1533 # and that '&' is escaped.
1535 self.failUnlessIn(entity, self._htmlname_raw)
1536 self.failIfIn(entity, self._htmlname_escaped)
1537 self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1538 self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1539 d.addCallback(_check)
1542 def test_GET_root_html(self):
1544 d.addCallback(self._check_upload_and_mkdir_forms)
1547 def test_GET_DIRURL(self):
1548 # the addSlash means we get a redirect here
1549 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1551 d = self.GET(self.public_url + "/foo", followRedirect=True)
1553 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1555 # the FILE reference points to a URI, but it should end in bar.txt
1556 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1557 (ROOT, urllib.quote(self._bar_txt_uri)))
1558 get_bar = "".join([r'<td>FILE</td>',
1560 r'<a href="%s">bar.txt</a>' % bar_url,
1562 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1564 self.failUnless(re.search(get_bar, res), res)
1565 for label in ['unlink', 'rename/relink']:
1566 for line in res.split("\n"):
1567 # find the line that contains the relevant button for bar.txt
1568 if ("form action" in line and
1569 ('value="%s"' % (label,)) in line and
1570 'value="bar.txt"' in line):
1571 # the form target should use a relative URL
1572 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1573 self.failUnlessIn('action="%s"' % foo_url, line)
1574 # and the when_done= should too
1575 #done_url = urllib.quote(???)
1576 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1578 # 'unlink' needs to use POST because it directly has a side effect
1579 if label == 'unlink':
1580 self.failUnlessIn('method="post"', line)
1583 self.fail("unable to find '%s bar.txt' line" % (label,))
1585 # the DIR reference just points to a URI
1586 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1587 get_sub = ((r'<td>DIR</td>')
1588 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1589 self.failUnless(re.search(get_sub, res), res)
1590 d.addCallback(_check)
1592 # look at a readonly directory
1593 d.addCallback(lambda res:
1594 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1596 self.failUnlessIn("(read-only)", res)
1597 self.failIfIn("Upload a file", res)
1598 d.addCallback(_check2)
1600 # and at a directory that contains a readonly directory
1601 d.addCallback(lambda res:
1602 self.GET(self.public_url, followRedirect=True))
1604 self.failUnless(re.search('<td>DIR-RO</td>'
1605 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1606 d.addCallback(_check3)
1608 # and an empty directory
1609 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1611 self.failUnlessIn("directory is empty", res)
1612 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1613 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1614 d.addCallback(_check4)
1616 # and at a literal directory
1617 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1618 d.addCallback(lambda res:
1619 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1621 self.failUnlessIn('(immutable)', res)
1622 self.failUnless(re.search('<td>FILE</td>'
1623 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1624 d.addCallback(_check5)
1627 def test_GET_DIRURL_badtype(self):
1628 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1632 self.public_url + "/foo?t=bogus")
1635 def test_GET_DIRURL_json(self):
1636 d = self.GET(self.public_url + "/foo?t=json")
1637 d.addCallback(self.failUnlessIsFooJSON)
1640 def test_GET_DIRURL_json_format(self):
1641 d = self.PUT(self.public_url + \
1642 "/foo/sdmf.txt?format=sdmf",
1643 self.NEWFILE_CONTENTS * 300000)
1644 d.addCallback(lambda ignored:
1645 self.PUT(self.public_url + \
1646 "/foo/mdmf.txt?format=mdmf",
1647 self.NEWFILE_CONTENTS * 300000))
1648 # Now we have an MDMF and SDMF file in the directory. If we GET
1649 # its JSON, we should see their encodings.
1650 d.addCallback(lambda ignored:
1651 self.GET(self.public_url + "/foo?t=json"))
1652 def _got_json(json):
1653 data = simplejson.loads(json)
1654 assert data[0] == "dirnode"
1657 kids = data['children']
1659 mdmf_data = kids['mdmf.txt'][1]
1660 self.failUnlessIn("format", mdmf_data)
1661 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1663 sdmf_data = kids['sdmf.txt'][1]
1664 self.failUnlessIn("format", sdmf_data)
1665 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1666 d.addCallback(_got_json)
1670 def test_POST_DIRURL_manifest_no_ophandle(self):
1671 d = self.shouldFail2(error.Error,
1672 "test_POST_DIRURL_manifest_no_ophandle",
1674 "slow operation requires ophandle=",
1675 self.POST, self.public_url, t="start-manifest")
1678 def test_POST_DIRURL_manifest(self):
1679 d = defer.succeed(None)
1680 def getman(ignored, output):
1681 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1682 followRedirect=True)
1683 d.addCallback(self.wait_for_operation, "125")
1684 d.addCallback(self.get_operation_results, "125", output)
1686 d.addCallback(getman, None)
1687 def _got_html(manifest):
1688 self.failUnlessIn("Manifest of SI=", manifest)
1689 self.failUnlessIn("<td>sub</td>", manifest)
1690 self.failUnlessIn(self._sub_uri, manifest)
1691 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1692 self.failUnlessIn(FAVICON_MARKUP, manifest)
1693 d.addCallback(_got_html)
1695 # both t=status and unadorned GET should be identical
1696 d.addCallback(lambda res: self.GET("/operations/125"))
1697 d.addCallback(_got_html)
1699 d.addCallback(getman, "html")
1700 d.addCallback(_got_html)
1701 d.addCallback(getman, "text")
1702 def _got_text(manifest):
1703 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1704 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1705 d.addCallback(_got_text)
1706 d.addCallback(getman, "JSON")
1708 data = res["manifest"]
1710 for (path_list, cap) in data:
1711 got[tuple(path_list)] = cap
1712 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1713 self.failUnlessIn((u"sub", u"baz.txt"), got)
1714 self.failUnlessIn("finished", res)
1715 self.failUnlessIn("origin", res)
1716 self.failUnlessIn("storage-index", res)
1717 self.failUnlessIn("verifycaps", res)
1718 self.failUnlessIn("stats", res)
1719 d.addCallback(_got_json)
1722 def test_POST_DIRURL_deepsize_no_ophandle(self):
1723 d = self.shouldFail2(error.Error,
1724 "test_POST_DIRURL_deepsize_no_ophandle",
1726 "slow operation requires ophandle=",
1727 self.POST, self.public_url, t="start-deep-size")
1730 def test_POST_DIRURL_deepsize(self):
1731 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1732 followRedirect=True)
1733 d.addCallback(self.wait_for_operation, "126")
1734 d.addCallback(self.get_operation_results, "126", "json")
1735 def _got_json(data):
1736 self.failUnlessReallyEqual(data["finished"], True)
1738 self.failUnless(size > 1000)
1739 d.addCallback(_got_json)
1740 d.addCallback(self.get_operation_results, "126", "text")
1742 mo = re.search(r'^size: (\d+)$', res, re.M)
1743 self.failUnless(mo, res)
1744 size = int(mo.group(1))
1745 # with directories, the size varies.
1746 self.failUnless(size > 1000)
1747 d.addCallback(_got_text)
1750 def test_POST_DIRURL_deepstats_no_ophandle(self):
1751 d = self.shouldFail2(error.Error,
1752 "test_POST_DIRURL_deepstats_no_ophandle",
1754 "slow operation requires ophandle=",
1755 self.POST, self.public_url, t="start-deep-stats")
1758 def test_POST_DIRURL_deepstats(self):
1759 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1760 followRedirect=True)
1761 d.addCallback(self.wait_for_operation, "127")
1762 d.addCallback(self.get_operation_results, "127", "json")
1763 def _got_json(stats):
1764 expected = {"count-immutable-files": 4,
1765 "count-mutable-files": 2,
1766 "count-literal-files": 0,
1768 "count-directories": 3,
1769 "size-immutable-files": 76,
1770 "size-literal-files": 0,
1771 #"size-directories": 1912, # varies
1772 #"largest-directory": 1590,
1773 "largest-directory-children": 8,
1774 "largest-immutable-file": 19,
1776 for k,v in expected.iteritems():
1777 self.failUnlessReallyEqual(stats[k], v,
1778 "stats[%s] was %s, not %s" %
1780 self.failUnlessReallyEqual(stats["size-files-histogram"],
1782 d.addCallback(_got_json)
1785 def test_POST_DIRURL_stream_manifest(self):
1786 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1788 self.failUnless(res.endswith("\n"))
1789 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1790 self.failUnlessReallyEqual(len(units), 10)
1791 self.failUnlessEqual(units[-1]["type"], "stats")
1793 self.failUnlessEqual(first["path"], [])
1794 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1795 self.failUnlessEqual(first["type"], "directory")
1796 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1797 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1798 self.failIfEqual(baz["storage-index"], None)
1799 self.failIfEqual(baz["verifycap"], None)
1800 self.failIfEqual(baz["repaircap"], None)
1801 # XXX: Add quux and baz to this test.
1803 d.addCallback(_check)
1806 def test_GET_DIRURL_uri(self):
1807 d = self.GET(self.public_url + "/foo?t=uri")
1809 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1810 d.addCallback(_check)
1813 def test_GET_DIRURL_readonly_uri(self):
1814 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1816 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1817 d.addCallback(_check)
1820 def test_PUT_NEWDIRURL(self):
1821 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1822 d.addCallback(lambda res:
1823 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1824 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1825 d.addCallback(self.failUnlessNodeKeysAre, [])
1828 def test_PUT_NEWDIRURL_mdmf(self):
1829 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1830 d.addCallback(lambda res:
1831 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1832 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1833 d.addCallback(lambda node:
1834 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1837 def test_PUT_NEWDIRURL_sdmf(self):
1838 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1840 d.addCallback(lambda res:
1841 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1842 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1843 d.addCallback(lambda node:
1844 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1847 def test_PUT_NEWDIRURL_bad_format(self):
1848 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1849 400, "Bad Request", "Unknown format: foo",
1850 self.PUT, self.public_url +
1851 "/foo/newdir=?t=mkdir&format=foo", "")
1853 def test_POST_NEWDIRURL(self):
1854 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1855 d.addCallback(lambda res:
1856 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1857 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1858 d.addCallback(self.failUnlessNodeKeysAre, [])
1861 def test_POST_NEWDIRURL_mdmf(self):
1862 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1863 d.addCallback(lambda res:
1864 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1865 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1866 d.addCallback(lambda node:
1867 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1870 def test_POST_NEWDIRURL_sdmf(self):
1871 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1872 d.addCallback(lambda res:
1873 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1874 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875 d.addCallback(lambda node:
1876 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1879 def test_POST_NEWDIRURL_bad_format(self):
1880 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1881 400, "Bad Request", "Unknown format: foo",
1882 self.POST2, self.public_url + \
1883 "/foo/newdir?t=mkdir&format=foo", "")
1885 def test_POST_NEWDIRURL_emptyname(self):
1886 # an empty pathname component (i.e. a double-slash) is disallowed
1887 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1889 "The webapi does not allow empty pathname components, i.e. a double slash",
1890 self.POST, self.public_url + "//?t=mkdir")
1893 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1894 (newkids, caps) = self._create_initial_children()
1895 query = "/foo/newdir?t=mkdir-with-children"
1896 if version == MDMF_VERSION:
1897 query += "&format=mdmf"
1898 elif version == SDMF_VERSION:
1899 query += "&format=sdmf"
1901 version = SDMF_VERSION # for later
1902 d = self.POST2(self.public_url + query,
1903 simplejson.dumps(newkids))
1905 n = self.s.create_node_from_uri(uri.strip())
1906 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1907 self.failUnlessEqual(n._node.get_version(), version)
1908 d2.addCallback(lambda ign:
1909 self.failUnlessROChildURIIs(n, u"child-imm",
1911 d2.addCallback(lambda ign:
1912 self.failUnlessRWChildURIIs(n, u"child-mutable",
1914 d2.addCallback(lambda ign:
1915 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1917 d2.addCallback(lambda ign:
1918 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1919 caps['unknown_rocap']))
1920 d2.addCallback(lambda ign:
1921 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1922 caps['unknown_rwcap']))
1923 d2.addCallback(lambda ign:
1924 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1925 caps['unknown_immcap']))
1926 d2.addCallback(lambda ign:
1927 self.failUnlessRWChildURIIs(n, u"dirchild",
1929 d2.addCallback(lambda ign:
1930 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1932 d2.addCallback(lambda ign:
1933 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1934 caps['emptydircap']))
1936 d.addCallback(_check)
1937 d.addCallback(lambda res:
1938 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1939 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1940 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1941 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1942 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1945 def test_POST_NEWDIRURL_initial_children(self):
1946 return self._do_POST_NEWDIRURL_initial_children_test()
1948 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1949 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1951 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1952 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1954 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1955 (newkids, caps) = self._create_initial_children()
1956 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1957 400, "Bad Request", "Unknown format: foo",
1958 self.POST2, self.public_url + \
1959 "/foo/newdir?t=mkdir-with-children&format=foo",
1960 simplejson.dumps(newkids))
1962 def test_POST_NEWDIRURL_immutable(self):
1963 (newkids, caps) = self._create_immutable_children()
1964 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1965 simplejson.dumps(newkids))
1967 n = self.s.create_node_from_uri(uri.strip())
1968 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1969 d2.addCallback(lambda ign:
1970 self.failUnlessROChildURIIs(n, u"child-imm",
1972 d2.addCallback(lambda ign:
1973 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1974 caps['unknown_immcap']))
1975 d2.addCallback(lambda ign:
1976 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1978 d2.addCallback(lambda ign:
1979 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1981 d2.addCallback(lambda ign:
1982 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1983 caps['emptydircap']))
1985 d.addCallback(_check)
1986 d.addCallback(lambda res:
1987 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1988 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1990 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1991 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1992 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1993 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1994 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1995 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1996 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1997 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1998 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1999 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2000 d.addErrback(self.explain_web_error)
2003 def test_POST_NEWDIRURL_immutable_bad(self):
2004 (newkids, caps) = self._create_initial_children()
2005 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2007 "needed to be immutable but was not",
2009 self.public_url + "/foo/newdir?t=mkdir-immutable",
2010 simplejson.dumps(newkids))
2013 def test_PUT_NEWDIRURL_exists(self):
2014 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2015 d.addCallback(lambda res:
2016 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2017 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2018 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2021 def test_PUT_NEWDIRURL_blocked(self):
2022 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2023 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2025 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2026 d.addCallback(lambda res:
2027 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2028 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2029 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2032 def test_PUT_NEWDIRURL_mkdirs(self):
2033 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2034 d.addCallback(lambda res:
2035 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2036 d.addCallback(lambda res:
2037 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2038 d.addCallback(lambda res:
2039 self._foo_node.get_child_at_path(u"subdir/newdir"))
2040 d.addCallback(self.failUnlessNodeKeysAre, [])
2043 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2044 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2045 d.addCallback(lambda ignored:
2046 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2047 d.addCallback(lambda ignored:
2048 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2049 d.addCallback(lambda ignored:
2050 self._foo_node.get_child_at_path(u"subdir"))
2051 def _got_subdir(subdir):
2052 # XXX: What we want?
2053 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2054 self.failUnlessNodeHasChild(subdir, u"newdir")
2055 return subdir.get_child_at_path(u"newdir")
2056 d.addCallback(_got_subdir)
2057 d.addCallback(lambda newdir:
2058 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2061 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2062 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2063 d.addCallback(lambda ignored:
2064 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2065 d.addCallback(lambda ignored:
2066 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2067 d.addCallback(lambda ignored:
2068 self._foo_node.get_child_at_path(u"subdir"))
2069 def _got_subdir(subdir):
2070 # XXX: What we want?
2071 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2072 self.failUnlessNodeHasChild(subdir, u"newdir")
2073 return subdir.get_child_at_path(u"newdir")
2074 d.addCallback(_got_subdir)
2075 d.addCallback(lambda newdir:
2076 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2079 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2080 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2081 400, "Bad Request", "Unknown format: foo",
2082 self.PUT, self.public_url + \
2083 "/foo/subdir/newdir?t=mkdir&format=foo",
2086 def test_DELETE_DIRURL(self):
2087 d = self.DELETE(self.public_url + "/foo")
2088 d.addCallback(lambda res:
2089 self.failIfNodeHasChild(self.public_root, u"foo"))
2092 def test_DELETE_DIRURL_missing(self):
2093 d = self.DELETE(self.public_url + "/foo/missing")
2094 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2095 d.addCallback(lambda res:
2096 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2099 def test_DELETE_DIRURL_missing2(self):
2100 d = self.DELETE(self.public_url + "/missing")
2101 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2104 def dump_root(self):
2106 w = webish.DirnodeWalkerMixin()
2107 def visitor(childpath, childnode, metadata):
2109 d = w.walk(self.public_root, visitor)
2112 def failUnlessNodeKeysAre(self, node, expected_keys):
2113 for k in expected_keys:
2114 assert isinstance(k, unicode)
2116 def _check(children):
2117 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2118 d.addCallback(_check)
2120 def failUnlessNodeHasChild(self, node, name):
2121 assert isinstance(name, unicode)
2123 def _check(children):
2124 self.failUnlessIn(name, children)
2125 d.addCallback(_check)
2127 def failIfNodeHasChild(self, node, name):
2128 assert isinstance(name, unicode)
2130 def _check(children):
2131 self.failIfIn(name, children)
2132 d.addCallback(_check)
2135 def failUnlessChildContentsAre(self, node, name, expected_contents):
2136 assert isinstance(name, unicode)
2137 d = node.get_child_at_path(name)
2138 d.addCallback(lambda node: download_to_data(node))
2139 def _check(contents):
2140 self.failUnlessReallyEqual(contents, expected_contents)
2141 d.addCallback(_check)
2144 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2145 assert isinstance(name, unicode)
2146 d = node.get_child_at_path(name)
2147 d.addCallback(lambda node: node.download_best_version())
2148 def _check(contents):
2149 self.failUnlessReallyEqual(contents, expected_contents)
2150 d.addCallback(_check)
2153 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2154 assert isinstance(name, unicode)
2155 d = node.get_child_at_path(name)
2157 self.failUnless(child.is_unknown() or not child.is_readonly())
2158 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2159 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2160 expected_ro_uri = self._make_readonly(expected_uri)
2162 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2163 d.addCallback(_check)
2166 def failUnlessROChildURIIs(self, node, name, expected_uri):
2167 assert isinstance(name, unicode)
2168 d = node.get_child_at_path(name)
2170 self.failUnless(child.is_unknown() or child.is_readonly())
2171 self.failUnlessReallyEqual(child.get_write_uri(), None)
2172 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2173 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2174 d.addCallback(_check)
2177 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2178 assert isinstance(name, unicode)
2179 d = node.get_child_at_path(name)
2181 self.failUnless(child.is_unknown() or not child.is_readonly())
2182 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2183 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2184 expected_ro_uri = self._make_readonly(got_uri)
2186 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2187 d.addCallback(_check)
2190 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2191 assert isinstance(name, unicode)
2192 d = node.get_child_at_path(name)
2194 self.failUnless(child.is_unknown() or child.is_readonly())
2195 self.failUnlessReallyEqual(child.get_write_uri(), None)
2196 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2197 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2198 d.addCallback(_check)
2201 def failUnlessCHKURIHasContents(self, got_uri, contents):
2202 self.failUnless(self.get_all_contents()[got_uri] == contents)
2204 def test_POST_upload(self):
2205 d = self.POST(self.public_url + "/foo", t="upload",
2206 file=("new.txt", self.NEWFILE_CONTENTS))
2208 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2209 d.addCallback(lambda res:
2210 self.failUnlessChildContentsAre(fn, u"new.txt",
2211 self.NEWFILE_CONTENTS))
2214 def test_POST_upload_unicode(self):
2215 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2216 d = self.POST(self.public_url + "/foo", t="upload",
2217 file=(filename, self.NEWFILE_CONTENTS))
2219 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2220 d.addCallback(lambda res:
2221 self.failUnlessChildContentsAre(fn, filename,
2222 self.NEWFILE_CONTENTS))
2223 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2224 d.addCallback(lambda res: self.GET(target_url))
2225 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2226 self.NEWFILE_CONTENTS,
2230 def test_POST_upload_unicode_named(self):
2231 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2232 d = self.POST(self.public_url + "/foo", t="upload",
2234 file=("overridden", self.NEWFILE_CONTENTS))
2236 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2237 d.addCallback(lambda res:
2238 self.failUnlessChildContentsAre(fn, filename,
2239 self.NEWFILE_CONTENTS))
2240 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2241 d.addCallback(lambda res: self.GET(target_url))
2242 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2243 self.NEWFILE_CONTENTS,
2247 def test_POST_upload_no_link(self):
2248 d = self.POST("/uri", t="upload",
2249 file=("new.txt", self.NEWFILE_CONTENTS))
2250 def _check_upload_results(page):
2251 # this should be a page which describes the results of the upload
2252 # that just finished.
2253 self.failUnlessIn("Upload Results:", page)
2254 self.failUnlessIn("URI:", page)
2255 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2256 mo = uri_re.search(page)
2257 self.failUnless(mo, page)
2258 new_uri = mo.group(1)
2260 d.addCallback(_check_upload_results)
2261 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2264 def test_POST_upload_no_link_whendone(self):
2265 d = self.POST("/uri", t="upload", when_done="/",
2266 file=("new.txt", self.NEWFILE_CONTENTS))
2267 d.addBoth(self.shouldRedirect, "/")
2270 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2271 d = defer.maybeDeferred(callable, *args, **kwargs)
2273 if isinstance(res, failure.Failure):
2274 res.trap(error.PageRedirect)
2275 statuscode = res.value.status
2276 target = res.value.location
2277 return checker(statuscode, target)
2278 self.fail("%s: callable was supposed to redirect, not return '%s'"
2283 def test_POST_upload_no_link_whendone_results(self):
2284 def check(statuscode, target):
2285 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2286 self.failUnless(target.startswith(self.webish_url), target)
2287 return client.getPage(target, method="GET")
2288 # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2289 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2291 self.POST, "/uri", t="upload",
2292 when_done="/%75ri/%(uri)s",
2293 file=("new.txt", self.NEWFILE_CONTENTS))
2294 d.addCallback(lambda res:
2295 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2298 def test_POST_upload_no_link_mutable(self):
2299 d = self.POST("/uri", t="upload", mutable="true",
2300 file=("new.txt", self.NEWFILE_CONTENTS))
2301 def _check(filecap):
2302 filecap = filecap.strip()
2303 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2304 self.filecap = filecap
2305 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2306 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2307 n = self.s.create_node_from_uri(filecap)
2308 return n.download_best_version()
2309 d.addCallback(_check)
2311 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2312 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2313 d.addCallback(_check2)
2315 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2316 return self.GET("/file/%s" % urllib.quote(self.filecap))
2317 d.addCallback(_check3)
2319 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2320 d.addCallback(_check4)
2323 def test_POST_upload_no_link_mutable_toobig(self):
2324 # The SDMF size limit is no longer in place, so we should be
2325 # able to upload mutable files that are as large as we want them
2327 d = self.POST("/uri", t="upload", mutable="true",
2328 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2332 def test_POST_upload_format_unlinked(self):
2333 def _check_upload_unlinked(ign, format, uri_prefix):
2334 filename = format + ".txt"
2335 d = self.POST("/uri?t=upload&format=" + format,
2336 file=(filename, self.NEWFILE_CONTENTS * 300000))
2337 def _got_results(results):
2338 if format.upper() in ("SDMF", "MDMF"):
2339 # webapi.rst says this returns a filecap
2342 # for immutable, it returns an "upload results page", and
2343 # the filecap is buried inside
2344 line = [l for l in results.split("\n") if "URI: " in l][0]
2345 mo = re.search(r'<span>([^<]+)</span>', line)
2346 filecap = mo.group(1)
2347 self.failUnless(filecap.startswith(uri_prefix),
2348 (uri_prefix, filecap))
2349 return self.GET("/uri/%s?t=json" % filecap)
2350 d.addCallback(_got_results)
2351 def _got_json(json):
2352 data = simplejson.loads(json)
2354 self.failUnlessIn("format", data)
2355 self.failUnlessEqual(data["format"], format.upper())
2356 d.addCallback(_got_json)
2358 d = defer.succeed(None)
2359 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2360 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2361 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2362 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2365 def test_POST_upload_bad_format_unlinked(self):
2366 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2367 400, "Bad Request", "Unknown format: foo",
2369 "/uri?t=upload&format=foo",
2370 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2372 def test_POST_upload_format(self):
2373 def _check_upload(ign, format, uri_prefix, fn=None):
2374 filename = format + ".txt"
2375 d = self.POST(self.public_url +
2376 "/foo?t=upload&format=" + format,
2377 file=(filename, self.NEWFILE_CONTENTS * 300000))
2378 def _got_filecap(filecap):
2380 filenameu = unicode(filename)
2381 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2382 self.failUnless(filecap.startswith(uri_prefix))
2383 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2384 d.addCallback(_got_filecap)
2385 def _got_json(json):
2386 data = simplejson.loads(json)
2388 self.failUnlessIn("format", data)
2389 self.failUnlessEqual(data["format"], format.upper())
2390 d.addCallback(_got_json)
2393 d = defer.succeed(None)
2394 d.addCallback(_check_upload, "chk", "URI:CHK")
2395 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2396 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2397 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2400 def test_POST_upload_bad_format(self):
2401 return self.shouldHTTPError("POST_upload_bad_format",
2402 400, "Bad Request", "Unknown format: foo",
2403 self.POST, self.public_url + \
2404 "/foo?t=upload&format=foo",
2405 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2407 def test_POST_upload_mutable(self):
2408 # this creates a mutable file
2409 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2410 file=("new.txt", self.NEWFILE_CONTENTS))
2412 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2413 d.addCallback(lambda res:
2414 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2415 self.NEWFILE_CONTENTS))
2416 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2418 self.failUnless(IMutableFileNode.providedBy(newnode))
2419 self.failUnless(newnode.is_mutable())
2420 self.failIf(newnode.is_readonly())
2421 self._mutable_node = newnode
2422 self._mutable_uri = newnode.get_uri()
2425 # now upload it again and make sure that the URI doesn't change
2426 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2427 d.addCallback(lambda res:
2428 self.POST(self.public_url + "/foo", t="upload",
2430 file=("new.txt", NEWER_CONTENTS)))
2431 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2432 d.addCallback(lambda res:
2433 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2435 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2437 self.failUnless(IMutableFileNode.providedBy(newnode))
2438 self.failUnless(newnode.is_mutable())
2439 self.failIf(newnode.is_readonly())
2440 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2441 d.addCallback(_got2)
2443 # upload a second time, using PUT instead of POST
2444 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2445 d.addCallback(lambda res:
2446 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2447 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2448 d.addCallback(lambda res:
2449 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2452 # finally list the directory, since mutable files are displayed
2453 # slightly differently
2455 d.addCallback(lambda res:
2456 self.GET(self.public_url + "/foo/",
2457 followRedirect=True))
2458 def _check_page(res):
2459 # TODO: assert more about the contents
2460 self.failUnlessIn("SSK", res)
2462 d.addCallback(_check_page)
2464 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2466 self.failUnless(IMutableFileNode.providedBy(newnode))
2467 self.failUnless(newnode.is_mutable())
2468 self.failIf(newnode.is_readonly())
2469 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2470 d.addCallback(_got3)
2472 # look at the JSON form of the enclosing directory
2473 d.addCallback(lambda res:
2474 self.GET(self.public_url + "/foo/?t=json",
2475 followRedirect=True))
2476 def _check_page_json(res):
2477 parsed = simplejson.loads(res)
2478 self.failUnlessEqual(parsed[0], "dirnode")
2479 children = dict( [(unicode(name),value)
2481 in parsed[1]["children"].iteritems()] )
2482 self.failUnlessIn(u"new.txt", children)
2483 new_json = children[u"new.txt"]
2484 self.failUnlessEqual(new_json[0], "filenode")
2485 self.failUnless(new_json[1]["mutable"])
2486 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2487 ro_uri = self._mutable_node.get_readonly().to_string()
2488 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2489 d.addCallback(_check_page_json)
2491 # and the JSON form of the file
2492 d.addCallback(lambda res:
2493 self.GET(self.public_url + "/foo/new.txt?t=json"))
2494 def _check_file_json(res):
2495 parsed = simplejson.loads(res)
2496 self.failUnlessEqual(parsed[0], "filenode")
2497 self.failUnless(parsed[1]["mutable"])
2498 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2499 ro_uri = self._mutable_node.get_readonly().to_string()
2500 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2501 d.addCallback(_check_file_json)
2503 # and look at t=uri and t=readonly-uri
2504 d.addCallback(lambda res:
2505 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2506 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2507 d.addCallback(lambda res:
2508 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2509 def _check_ro_uri(res):
2510 ro_uri = self._mutable_node.get_readonly().to_string()
2511 self.failUnlessReallyEqual(res, ro_uri)
2512 d.addCallback(_check_ro_uri)
2514 # make sure we can get to it from /uri/URI
2515 d.addCallback(lambda res:
2516 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2517 d.addCallback(lambda res:
2518 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2520 # and that HEAD computes the size correctly
2521 d.addCallback(lambda res:
2522 self.HEAD(self.public_url + "/foo/new.txt",
2523 return_response=True))
2524 def _got_headers((res, status, headers)):
2525 self.failUnlessReallyEqual(res, "")
2526 self.failUnlessReallyEqual(headers["content-length"][0],
2527 str(len(NEW2_CONTENTS)))
2528 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2529 d.addCallback(_got_headers)
2531 # make sure that outdated size limits aren't enforced anymore.
2532 d.addCallback(lambda ignored:
2533 self.POST(self.public_url + "/foo", t="upload",
2536 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2537 d.addErrback(self.dump_error)
2540 def test_POST_upload_mutable_toobig(self):
2541 # SDMF had a size limti that was removed a while ago. MDMF has
2542 # never had a size limit. Test to make sure that we do not
2543 # encounter errors when trying to upload large mutable files,
2544 # since there should be no coded prohibitions regarding large
2546 d = self.POST(self.public_url + "/foo",
2547 t="upload", mutable="true",
2548 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2551 def dump_error(self, f):
2552 # if the web server returns an error code (like 400 Bad Request),
2553 # web.client.getPage puts the HTTP response body into the .response
2554 # attribute of the exception object that it gives back. It does not
2555 # appear in the Failure's repr(), so the ERROR that trial displays
2556 # will be rather terse and unhelpful. addErrback this method to the
2557 # end of your chain to get more information out of these errors.
2558 if f.check(error.Error):
2559 print "web.error.Error:"
2561 print f.value.response
2564 def test_POST_upload_replace(self):
2565 d = self.POST(self.public_url + "/foo", t="upload",
2566 file=("bar.txt", self.NEWFILE_CONTENTS))
2568 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2569 d.addCallback(lambda res:
2570 self.failUnlessChildContentsAre(fn, u"bar.txt",
2571 self.NEWFILE_CONTENTS))
2574 def test_POST_upload_no_replace_ok(self):
2575 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2576 file=("new.txt", self.NEWFILE_CONTENTS))
2577 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2578 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2579 self.NEWFILE_CONTENTS))
2582 def test_POST_upload_no_replace_queryarg(self):
2583 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2584 file=("bar.txt", self.NEWFILE_CONTENTS))
2585 d.addBoth(self.shouldFail, error.Error,
2586 "POST_upload_no_replace_queryarg",
2588 "There was already a child by that name, and you asked me "
2589 "to not replace it")
2590 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2591 d.addCallback(self.failUnlessIsBarDotTxt)
2594 def test_POST_upload_no_replace_field(self):
2595 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2596 file=("bar.txt", self.NEWFILE_CONTENTS))
2597 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2599 "There was already a child by that name, and you asked me "
2600 "to not replace it")
2601 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2602 d.addCallback(self.failUnlessIsBarDotTxt)
2605 def test_POST_upload_whendone(self):
2606 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2607 file=("new.txt", self.NEWFILE_CONTENTS))
2608 d.addBoth(self.shouldRedirect, "/THERE")
2610 d.addCallback(lambda res:
2611 self.failUnlessChildContentsAre(fn, u"new.txt",
2612 self.NEWFILE_CONTENTS))
2615 def test_POST_upload_named(self):
2617 d = self.POST(self.public_url + "/foo", t="upload",
2618 name="new.txt", file=self.NEWFILE_CONTENTS)
2619 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2620 d.addCallback(lambda res:
2621 self.failUnlessChildContentsAre(fn, u"new.txt",
2622 self.NEWFILE_CONTENTS))
2625 def test_POST_upload_named_badfilename(self):
2626 d = self.POST(self.public_url + "/foo", t="upload",
2627 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2628 d.addBoth(self.shouldFail, error.Error,
2629 "test_POST_upload_named_badfilename",
2631 "name= may not contain a slash",
2633 # make sure that nothing was added
2634 d.addCallback(lambda res:
2635 self.failUnlessNodeKeysAre(self._foo_node,
2636 [self._htmlname_unicode,
2637 u"bar.txt", u"baz.txt", u"blockingfile",
2638 u"empty", u"n\u00fc.txt", u"quux.txt",
2642 def test_POST_FILEURL_check(self):
2643 bar_url = self.public_url + "/foo/bar.txt"
2644 d = self.POST(bar_url, t="check")
2646 self.failUnlessIn("Healthy :", res)
2647 d.addCallback(_check)
2648 redir_url = "http://allmydata.org/TARGET"
2649 def _check2(statuscode, target):
2650 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2651 self.failUnlessReallyEqual(target, redir_url)
2652 d.addCallback(lambda res:
2653 self.shouldRedirect2("test_POST_FILEURL_check",
2657 when_done=redir_url))
2658 d.addCallback(lambda res:
2659 self.POST(bar_url, t="check", return_to=redir_url))
2661 self.failUnlessIn("Healthy :", res)
2662 self.failUnlessIn("Return to file", res)
2663 self.failUnlessIn(redir_url, res)
2664 d.addCallback(_check3)
2666 d.addCallback(lambda res:
2667 self.POST(bar_url, t="check", output="JSON"))
2668 def _check_json(res):
2669 data = simplejson.loads(res)
2670 self.failUnlessIn("storage-index", data)
2671 self.failUnless(data["results"]["healthy"])
2672 d.addCallback(_check_json)
2676 def test_POST_FILEURL_check_and_repair(self):
2677 bar_url = self.public_url + "/foo/bar.txt"
2678 d = self.POST(bar_url, t="check", repair="true")
2680 self.failUnlessIn("Healthy :", res)
2681 d.addCallback(_check)
2682 redir_url = "http://allmydata.org/TARGET"
2683 def _check2(statuscode, target):
2684 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2685 self.failUnlessReallyEqual(target, redir_url)
2686 d.addCallback(lambda res:
2687 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2690 t="check", repair="true",
2691 when_done=redir_url))
2692 d.addCallback(lambda res:
2693 self.POST(bar_url, t="check", return_to=redir_url))
2695 self.failUnlessIn("Healthy :", res)
2696 self.failUnlessIn("Return to file", res)
2697 self.failUnlessIn(redir_url, res)
2698 d.addCallback(_check3)
2701 def test_POST_DIRURL_check(self):
2702 foo_url = self.public_url + "/foo/"
2703 d = self.POST(foo_url, t="check")
2705 self.failUnlessIn("Healthy :", res)
2706 d.addCallback(_check)
2707 redir_url = "http://allmydata.org/TARGET"
2708 def _check2(statuscode, target):
2709 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2710 self.failUnlessReallyEqual(target, redir_url)
2711 d.addCallback(lambda res:
2712 self.shouldRedirect2("test_POST_DIRURL_check",
2716 when_done=redir_url))
2717 d.addCallback(lambda res:
2718 self.POST(foo_url, t="check", return_to=redir_url))
2720 self.failUnlessIn("Healthy :", res)
2721 self.failUnlessIn("Return to file/directory", res)
2722 self.failUnlessIn(redir_url, res)
2723 d.addCallback(_check3)
2725 d.addCallback(lambda res:
2726 self.POST(foo_url, t="check", output="JSON"))
2727 def _check_json(res):
2728 data = simplejson.loads(res)
2729 self.failUnlessIn("storage-index", data)
2730 self.failUnless(data["results"]["healthy"])
2731 d.addCallback(_check_json)
2735 def test_POST_DIRURL_check_and_repair(self):
2736 foo_url = self.public_url + "/foo/"
2737 d = self.POST(foo_url, t="check", repair="true")
2739 self.failUnlessIn("Healthy :", res)
2740 d.addCallback(_check)
2741 redir_url = "http://allmydata.org/TARGET"
2742 def _check2(statuscode, target):
2743 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2744 self.failUnlessReallyEqual(target, redir_url)
2745 d.addCallback(lambda res:
2746 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2749 t="check", repair="true",
2750 when_done=redir_url))
2751 d.addCallback(lambda res:
2752 self.POST(foo_url, t="check", return_to=redir_url))
2754 self.failUnlessIn("Healthy :", res)
2755 self.failUnlessIn("Return to file/directory", res)
2756 self.failUnlessIn(redir_url, res)
2757 d.addCallback(_check3)
2760 def test_POST_FILEURL_mdmf_check(self):
2761 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2762 d = self.POST(quux_url, t="check")
2764 self.failUnlessIn("Healthy", res)
2765 d.addCallback(_check)
2766 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2767 d.addCallback(lambda ignored:
2768 self.POST(quux_extension_url, t="check"))
2769 d.addCallback(_check)
2772 def test_POST_FILEURL_mdmf_check_and_repair(self):
2773 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2774 d = self.POST(quux_url, t="check", repair="true")
2776 self.failUnlessIn("Healthy", res)
2777 d.addCallback(_check)
2778 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2779 d.addCallback(lambda ignored:
2780 self.POST(quux_extension_url, t="check", repair="true"))
2781 d.addCallback(_check)
2784 def wait_for_operation(self, ignored, ophandle):
2785 url = "/operations/" + ophandle
2786 url += "?t=status&output=JSON"
2789 data = simplejson.loads(res)
2790 if not data["finished"]:
2791 d = self.stall(delay=1.0)
2792 d.addCallback(self.wait_for_operation, ophandle)
2798 def get_operation_results(self, ignored, ophandle, output=None):
2799 url = "/operations/" + ophandle
2802 url += "&output=" + output
2805 if output and output.lower() == "json":
2806 return simplejson.loads(res)
2811 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2812 d = self.shouldFail2(error.Error,
2813 "test_POST_DIRURL_deepcheck_no_ophandle",
2815 "slow operation requires ophandle=",
2816 self.POST, self.public_url, t="start-deep-check")
2819 def test_POST_DIRURL_deepcheck(self):
2820 def _check_redirect(statuscode, target):
2821 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2822 self.failUnless(target.endswith("/operations/123"))
2823 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2824 self.POST, self.public_url,
2825 t="start-deep-check", ophandle="123")
2826 d.addCallback(self.wait_for_operation, "123")
2827 def _check_json(data):
2828 self.failUnlessReallyEqual(data["finished"], True)
2829 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2830 self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2831 d.addCallback(_check_json)
2832 d.addCallback(self.get_operation_results, "123", "html")
2833 def _check_html(res):
2834 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2835 self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2836 self.failUnlessIn(FAVICON_MARKUP, res)
2837 d.addCallback(_check_html)
2839 d.addCallback(lambda res:
2840 self.GET("/operations/123/"))
2841 d.addCallback(_check_html) # should be the same as without the slash
2843 d.addCallback(lambda res:
2844 self.shouldFail2(error.Error, "one", "404 Not Found",
2845 "No detailed results for SI bogus",
2846 self.GET, "/operations/123/bogus"))
2848 foo_si = self._foo_node.get_storage_index()
2849 foo_si_s = base32.b2a(foo_si)
2850 d.addCallback(lambda res:
2851 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2852 def _check_foo_json(res):
2853 data = simplejson.loads(res)
2854 self.failUnlessEqual(data["storage-index"], foo_si_s)
2855 self.failUnless(data["results"]["healthy"])
2856 d.addCallback(_check_foo_json)
2859 def test_POST_DIRURL_deepcheck_and_repair(self):
2860 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2861 ophandle="124", output="json", followRedirect=True)
2862 d.addCallback(self.wait_for_operation, "124")
2863 def _check_json(data):
2864 self.failUnlessReallyEqual(data["finished"], True)
2865 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2866 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2867 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2868 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2869 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2870 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2871 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2872 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2873 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2874 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2875 d.addCallback(_check_json)
2876 d.addCallback(self.get_operation_results, "124", "html")
2877 def _check_html(res):
2878 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2880 self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2881 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2882 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2884 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2885 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2886 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2888 self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2889 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2890 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2892 self.failUnlessIn(FAVICON_MARKUP, res)
2893 d.addCallback(_check_html)
2896 def test_POST_FILEURL_bad_t(self):
2897 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2898 "POST to file: bad t=bogus",
2899 self.POST, self.public_url + "/foo/bar.txt",
2903 def test_POST_mkdir(self): # return value?
2904 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2905 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2906 d.addCallback(self.failUnlessNodeKeysAre, [])
2909 def test_POST_mkdir_mdmf(self):
2910 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2911 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2912 d.addCallback(lambda node:
2913 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2916 def test_POST_mkdir_sdmf(self):
2917 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2918 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2919 d.addCallback(lambda node:
2920 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2923 def test_POST_mkdir_bad_format(self):
2924 return self.shouldHTTPError("POST_mkdir_bad_format",
2925 400, "Bad Request", "Unknown format: foo",
2926 self.POST, self.public_url +
2927 "/foo?t=mkdir&name=newdir&format=foo")
2929 def test_POST_mkdir_initial_children(self):
2930 (newkids, caps) = self._create_initial_children()
2931 d = self.POST2(self.public_url +
2932 "/foo?t=mkdir-with-children&name=newdir",
2933 simplejson.dumps(newkids))
2934 d.addCallback(lambda res:
2935 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2936 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2938 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2939 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2942 def test_POST_mkdir_initial_children_mdmf(self):
2943 (newkids, caps) = self._create_initial_children()
2944 d = self.POST2(self.public_url +
2945 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2946 simplejson.dumps(newkids))
2947 d.addCallback(lambda res:
2948 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2949 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2950 d.addCallback(lambda node:
2951 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2952 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2953 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2958 def test_POST_mkdir_initial_children_sdmf(self):
2959 (newkids, caps) = self._create_initial_children()
2960 d = self.POST2(self.public_url +
2961 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2962 simplejson.dumps(newkids))
2963 d.addCallback(lambda res:
2964 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2965 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2966 d.addCallback(lambda node:
2967 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2968 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2969 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2973 def test_POST_mkdir_initial_children_bad_format(self):
2974 (newkids, caps) = self._create_initial_children()
2975 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2976 400, "Bad Request", "Unknown format: foo",
2977 self.POST, self.public_url + \
2978 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2979 simplejson.dumps(newkids))
2981 def test_POST_mkdir_immutable(self):
2982 (newkids, caps) = self._create_immutable_children()
2983 d = self.POST2(self.public_url +
2984 "/foo?t=mkdir-immutable&name=newdir",
2985 simplejson.dumps(newkids))
2986 d.addCallback(lambda res:
2987 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2988 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2989 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2990 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2991 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2992 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2993 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2994 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2995 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2996 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2997 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2998 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2999 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3002 def test_POST_mkdir_immutable_bad(self):
3003 (newkids, caps) = self._create_initial_children()
3004 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3006 "needed to be immutable but was not",
3009 "/foo?t=mkdir-immutable&name=newdir",
3010 simplejson.dumps(newkids))
3013 def test_POST_mkdir_2(self):
3014 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3015 d.addCallback(lambda res:
3016 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3017 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018 d.addCallback(self.failUnlessNodeKeysAre, [])
3021 def test_POST_mkdirs_2(self):
3022 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3023 d.addCallback(lambda res:
3024 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3025 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3026 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3027 d.addCallback(self.failUnlessNodeKeysAre, [])
3030 def test_POST_mkdir_no_parentdir_noredirect(self):
3031 d = self.POST("/uri?t=mkdir")
3032 def _after_mkdir(res):
3033 uri.DirectoryURI.init_from_string(res)
3034 d.addCallback(_after_mkdir)
3037 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3038 d = self.POST("/uri?t=mkdir&format=mdmf")
3039 def _after_mkdir(res):
3040 u = uri.from_string(res)
3041 # Check that this is an MDMF writecap
3042 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3043 d.addCallback(_after_mkdir)
3046 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3047 d = self.POST("/uri?t=mkdir&format=sdmf")
3048 def _after_mkdir(res):
3049 u = uri.from_string(res)
3050 self.failUnlessIsInstance(u, uri.DirectoryURI)
3051 d.addCallback(_after_mkdir)
3054 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3055 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3056 400, "Bad Request", "Unknown format: foo",
3057 self.POST, self.public_url +
3058 "/uri?t=mkdir&format=foo")
3060 def test_POST_mkdir_no_parentdir_noredirect2(self):
3061 # make sure form-based arguments (as on the welcome page) still work
3062 d = self.POST("/uri", t="mkdir")
3063 def _after_mkdir(res):
3064 uri.DirectoryURI.init_from_string(res)
3065 d.addCallback(_after_mkdir)
3066 d.addErrback(self.explain_web_error)
3069 def test_POST_mkdir_no_parentdir_redirect(self):
3070 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3071 d.addBoth(self.shouldRedirect, None, statuscode='303')
3072 def _check_target(target):
3073 target = urllib.unquote(target)
3074 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3075 d.addCallback(_check_target)
3078 def test_POST_mkdir_no_parentdir_redirect2(self):
3079 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3080 d.addBoth(self.shouldRedirect, None, statuscode='303')
3081 def _check_target(target):
3082 target = urllib.unquote(target)
3083 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3084 d.addCallback(_check_target)
3085 d.addErrback(self.explain_web_error)
3088 def _make_readonly(self, u):
3089 ro_uri = uri.from_string(u).get_readonly()
3092 return ro_uri.to_string()
3094 def _create_initial_children(self):
3095 contents, n, filecap1 = self.makefile(12)
3096 md1 = {"metakey1": "metavalue1"}
3097 filecap2 = make_mutable_file_uri()
3098 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3099 filecap3 = node3.get_readonly_uri()
3100 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3101 dircap = DirectoryNode(node4, None, None).get_uri()
3102 mdmfcap = make_mutable_file_uri(mdmf=True)
3103 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3104 emptydircap = "URI:DIR2-LIT:"
3105 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3106 "ro_uri": self._make_readonly(filecap1),
3107 "metadata": md1, }],
3108 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3109 "ro_uri": self._make_readonly(filecap2)}],
3110 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3111 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3112 "ro_uri": unknown_rocap}],
3113 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3114 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3115 u"dirchild": ["dirnode", {"rw_uri": dircap,
3116 "ro_uri": self._make_readonly(dircap)}],
3117 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3118 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3119 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3120 "ro_uri": self._make_readonly(mdmfcap)}],
3122 return newkids, {'filecap1': filecap1,
3123 'filecap2': filecap2,
3124 'filecap3': filecap3,
3125 'unknown_rwcap': unknown_rwcap,
3126 'unknown_rocap': unknown_rocap,
3127 'unknown_immcap': unknown_immcap,
3129 'litdircap': litdircap,
3130 'emptydircap': emptydircap,
3133 def _create_immutable_children(self):
3134 contents, n, filecap1 = self.makefile(12)
3135 md1 = {"metakey1": "metavalue1"}
3136 tnode = create_chk_filenode("immutable directory contents\n"*10,
3137 self.get_all_contents())
3138 dnode = DirectoryNode(tnode, None, None)
3139 assert not dnode.is_mutable()
3140 immdircap = dnode.get_uri()
3141 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3142 emptydircap = "URI:DIR2-LIT:"
3143 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3144 "metadata": md1, }],
3145 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3146 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3147 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3148 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3150 return newkids, {'filecap1': filecap1,
3151 'unknown_immcap': unknown_immcap,
3152 'immdircap': immdircap,
3153 'litdircap': litdircap,
3154 'emptydircap': emptydircap}
3156 def test_POST_mkdir_no_parentdir_initial_children(self):
3157 (newkids, caps) = self._create_initial_children()
3158 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3159 def _after_mkdir(res):
3160 self.failUnless(res.startswith("URI:DIR"), res)
3161 n = self.s.create_node_from_uri(res)
3162 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3163 d2.addCallback(lambda ign:
3164 self.failUnlessROChildURIIs(n, u"child-imm",
3166 d2.addCallback(lambda ign:
3167 self.failUnlessRWChildURIIs(n, u"child-mutable",
3169 d2.addCallback(lambda ign:
3170 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3172 d2.addCallback(lambda ign:
3173 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3174 caps['unknown_rwcap']))
3175 d2.addCallback(lambda ign:
3176 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3177 caps['unknown_rocap']))
3178 d2.addCallback(lambda ign:
3179 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3180 caps['unknown_immcap']))
3181 d2.addCallback(lambda ign:
3182 self.failUnlessRWChildURIIs(n, u"dirchild",
3185 d.addCallback(_after_mkdir)
3188 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3189 # the regular /uri?t=mkdir operation is specified to ignore its body.
3190 # Only t=mkdir-with-children pays attention to it.
3191 (newkids, caps) = self._create_initial_children()
3192 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3194 "t=mkdir does not accept children=, "
3195 "try t=mkdir-with-children instead",
3196 self.POST2, "/uri?t=mkdir", # without children
3197 simplejson.dumps(newkids))
3200 def test_POST_noparent_bad(self):
3201 d = self.shouldHTTPError("POST_noparent_bad",
3203 "/uri accepts only PUT, PUT?t=mkdir, "
3204 "POST?t=upload, and POST?t=mkdir",
3205 self.POST, "/uri?t=bogus")
3208 def test_POST_mkdir_no_parentdir_immutable(self):
3209 (newkids, caps) = self._create_immutable_children()
3210 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3211 def _after_mkdir(res):
3212 self.failUnless(res.startswith("URI:DIR"), res)
3213 n = self.s.create_node_from_uri(res)
3214 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3215 d2.addCallback(lambda ign:
3216 self.failUnlessROChildURIIs(n, u"child-imm",
3218 d2.addCallback(lambda ign:
3219 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3220 caps['unknown_immcap']))
3221 d2.addCallback(lambda ign:
3222 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3224 d2.addCallback(lambda ign:
3225 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3227 d2.addCallback(lambda ign:
3228 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3229 caps['emptydircap']))
3231 d.addCallback(_after_mkdir)
3234 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3235 (newkids, caps) = self._create_initial_children()
3236 d = self.shouldFail2(error.Error,
3237 "test_POST_mkdir_no_parentdir_immutable_bad",
3239 "needed to be immutable but was not",
3241 "/uri?t=mkdir-immutable",
3242 simplejson.dumps(newkids))
3245 def test_welcome_page_mkdir_button(self):
3246 # Fetch the welcome page.
3248 def _after_get_welcome_page(res):
3249 MKDIR_BUTTON_RE = re.compile(
3250 '<form action="([^"]*)" method="post".*'
3251 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3252 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3253 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3254 html = res.replace('\n', ' ')
3255 mo = MKDIR_BUTTON_RE.search(html)
3256 self.failUnless(mo, html)
3257 formaction = mo.group(1)
3259 formaname = mo.group(3)
3260 formavalue = mo.group(4)
3261 return (formaction, formt, formaname, formavalue)
3262 d.addCallback(_after_get_welcome_page)
3263 def _after_parse_form(res):
3264 (formaction, formt, formaname, formavalue) = res
3265 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3266 d.addCallback(_after_parse_form)
3267 d.addBoth(self.shouldRedirect, None, statuscode='303')
3270 def test_POST_mkdir_replace(self): # return value?
3271 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3272 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3273 d.addCallback(self.failUnlessNodeKeysAre, [])
3276 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3277 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3278 d.addBoth(self.shouldFail, error.Error,
3279 "POST_mkdir_no_replace_queryarg",
3281 "There was already a child by that name, and you asked me "
3282 "to not replace it")
3283 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3284 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3287 def test_POST_mkdir_no_replace_field(self): # return value?
3288 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3290 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3292 "There was already a child by that name, and you asked me "
3293 "to not replace it")
3294 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3295 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3298 def test_POST_mkdir_whendone_field(self):
3299 d = self.POST(self.public_url + "/foo",
3300 t="mkdir", name="newdir", when_done="/THERE")
3301 d.addBoth(self.shouldRedirect, "/THERE")
3302 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3303 d.addCallback(self.failUnlessNodeKeysAre, [])
3306 def test_POST_mkdir_whendone_queryarg(self):
3307 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3308 t="mkdir", name="newdir")
3309 d.addBoth(self.shouldRedirect, "/THERE")
3310 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3311 d.addCallback(self.failUnlessNodeKeysAre, [])
3314 def test_POST_bad_t(self):
3315 d = self.shouldFail2(error.Error, "POST_bad_t",
3317 "POST to a directory with bad t=BOGUS",
3318 self.POST, self.public_url + "/foo", t="BOGUS")
3321 def test_POST_set_children(self, command_name="set_children"):
3322 contents9, n9, newuri9 = self.makefile(9)
3323 contents10, n10, newuri10 = self.makefile(10)
3324 contents11, n11, newuri11 = self.makefile(11)
3327 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3330 "ctime": 1002777696.7564139,
3331 "mtime": 1002777696.7564139
3334 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3337 "ctime": 1002777696.7564139,
3338 "mtime": 1002777696.7564139
3341 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3344 "ctime": 1002777696.7564139,
3345 "mtime": 1002777696.7564139
3348 }""" % (newuri9, newuri10, newuri11)
3350 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3352 d = client.getPage(url, method="POST", postdata=reqbody)
3354 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3355 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3356 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3358 d.addCallback(_then)
3359 d.addErrback(self.dump_error)
3362 def test_POST_set_children_with_hyphen(self):
3363 return self.test_POST_set_children(command_name="set-children")
3365 def test_POST_link_uri(self):
3366 contents, n, newuri = self.makefile(8)
3367 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3368 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3369 d.addCallback(lambda res:
3370 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3374 def test_POST_link_uri_replace(self):
3375 contents, n, newuri = self.makefile(8)
3376 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3377 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3378 d.addCallback(lambda res:
3379 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3383 def test_POST_link_uri_unknown_bad(self):
3384 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3385 d.addBoth(self.shouldFail, error.Error,
3386 "POST_link_uri_unknown_bad",
3388 "unknown cap in a write slot")
3391 def test_POST_link_uri_unknown_ro_good(self):
3392 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3393 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3396 def test_POST_link_uri_unknown_imm_good(self):
3397 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3398 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3401 def test_POST_link_uri_no_replace_queryarg(self):
3402 contents, n, newuri = self.makefile(8)
3403 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3404 name="bar.txt", uri=newuri)
3405 d.addBoth(self.shouldFail, error.Error,
3406 "POST_link_uri_no_replace_queryarg",
3408 "There was already a child by that name, and you asked me "
3409 "to not replace it")
3410 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3411 d.addCallback(self.failUnlessIsBarDotTxt)
3414 def test_POST_link_uri_no_replace_field(self):
3415 contents, n, newuri = self.makefile(8)
3416 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3417 name="bar.txt", uri=newuri)
3418 d.addBoth(self.shouldFail, error.Error,
3419 "POST_link_uri_no_replace_field",
3421 "There was already a child by that name, and you asked me "
3422 "to not replace it")
3423 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3424 d.addCallback(self.failUnlessIsBarDotTxt)
3427 def test_POST_delete(self, command_name='delete'):
3428 d = self._foo_node.list()
3429 def _check_before(children):
3430 self.failUnlessIn(u"bar.txt", children)
3431 d.addCallback(_check_before)
3432 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3433 d.addCallback(lambda res: self._foo_node.list())
3434 def _check_after(children):
3435 self.failIfIn(u"bar.txt", children)
3436 d.addCallback(_check_after)
3439 def test_POST_unlink(self):
3440 return self.test_POST_delete(command_name='unlink')
3442 def test_POST_rename_file(self):
3443 d = self.POST(self.public_url + "/foo", t="rename",
3444 from_name="bar.txt", to_name='wibble.txt')
3445 d.addCallback(lambda res:
3446 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3447 d.addCallback(lambda res:
3448 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3449 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3450 d.addCallback(self.failUnlessIsBarDotTxt)
3451 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3452 d.addCallback(self.failUnlessIsBarJSON)
3455 def test_POST_rename_file_redundant(self):
3456 d = self.POST(self.public_url + "/foo", t="rename",
3457 from_name="bar.txt", to_name='bar.txt')
3458 d.addCallback(lambda res:
3459 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3460 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3461 d.addCallback(self.failUnlessIsBarDotTxt)
3462 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3463 d.addCallback(self.failUnlessIsBarJSON)
3466 def test_POST_rename_file_replace(self):
3467 # rename a file and replace a directory with it
3468 d = self.POST(self.public_url + "/foo", t="rename",
3469 from_name="bar.txt", to_name='empty')
3470 d.addCallback(lambda res:
3471 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3472 d.addCallback(lambda res:
3473 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3474 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3475 d.addCallback(self.failUnlessIsBarDotTxt)
3476 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3477 d.addCallback(self.failUnlessIsBarJSON)
3480 def test_POST_rename_file_no_replace_queryarg(self):
3481 # rename a file and replace a directory with it
3482 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3483 from_name="bar.txt", to_name='empty')
3484 d.addBoth(self.shouldFail, error.Error,
3485 "POST_rename_file_no_replace_queryarg",
3487 "There was already a child by that name, and you asked me "
3488 "to not replace it")
3489 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3490 d.addCallback(self.failUnlessIsEmptyJSON)
3493 def test_POST_rename_file_no_replace_field(self):
3494 # rename a file and replace a directory with it
3495 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3496 from_name="bar.txt", to_name='empty')
3497 d.addBoth(self.shouldFail, error.Error,
3498 "POST_rename_file_no_replace_field",
3500 "There was already a child by that name, and you asked me "
3501 "to not replace it")
3502 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3503 d.addCallback(self.failUnlessIsEmptyJSON)
3506 def test_POST_rename_file_no_replace_same_link(self):
3507 d = self.POST(self.public_url + "/foo", t="rename",
3508 replace="false", from_name="bar.txt", to_name="bar.txt")
3509 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3510 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3511 d.addCallback(self.failUnlessIsBarDotTxt)
3512 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3513 d.addCallback(self.failUnlessIsBarJSON)
3516 def test_POST_rename_file_replace_only_files(self):
3517 d = self.POST(self.public_url + "/foo", t="rename",
3518 replace="only-files", from_name="bar.txt",
3520 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3521 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3522 d.addCallback(self.failUnlessIsBarDotTxt)
3523 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3524 d.addCallback(self.failUnlessIsBarJSON)
3527 def test_POST_rename_file_replace_only_files_conflict(self):
3528 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3530 "There was already a child by that name, and you asked me to not replace it.",
3531 self.POST, self.public_url + "/foo", t="relink",
3532 replace="only-files", from_name="bar.txt",
3534 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3535 d.addCallback(self.failUnlessIsBarDotTxt)
3536 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3537 d.addCallback(self.failUnlessIsBarJSON)
3540 def failUnlessIsEmptyJSON(self, res):
3541 data = simplejson.loads(res)
3542 self.failUnlessEqual(data[0], "dirnode", data)
3543 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3545 def test_POST_rename_file_to_slash_fail(self):
3546 d = self.POST(self.public_url + "/foo", t="rename",
3547 from_name="bar.txt", to_name='kirk/spock.txt')
3548 d.addBoth(self.shouldFail, error.Error,
3549 "test_POST_rename_file_to_slash_fail",
3551 "to_name= may not contain a slash",
3553 d.addCallback(lambda res:
3554 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3557 def test_POST_rename_file_from_slash_fail(self):
3558 d = self.POST(self.public_url + "/foo", t="rename",
3559 from_name="sub/bar.txt", to_name='spock.txt')
3560 d.addBoth(self.shouldFail, error.Error,
3561 "test_POST_rename_from_file_slash_fail",
3563 "from_name= may not contain a slash",
3565 d.addCallback(lambda res:
3566 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3569 def test_POST_rename_dir(self):
3570 d = self.POST(self.public_url, t="rename",
3571 from_name="foo", to_name='plunk')
3572 d.addCallback(lambda res:
3573 self.failIfNodeHasChild(self.public_root, u"foo"))
3574 d.addCallback(lambda res:
3575 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3576 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3577 d.addCallback(self.failUnlessIsFooJSON)
3580 def test_POST_relink_file(self):
3581 d = self.POST(self.public_url + "/foo", t="relink",
3582 from_name="bar.txt",
3583 to_dir=self.public_root.get_uri() + "/foo/sub")
3584 d.addCallback(lambda res:
3585 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3586 d.addCallback(lambda res:
3587 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3588 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3589 d.addCallback(self.failUnlessIsBarDotTxt)
3590 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3591 d.addCallback(self.failUnlessIsBarJSON)
3594 def test_POST_relink_file_new_name(self):
3595 d = self.POST(self.public_url + "/foo", t="relink",
3596 from_name="bar.txt",
3597 to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3598 d.addCallback(lambda res:
3599 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3600 d.addCallback(lambda res:
3601 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3602 d.addCallback(lambda res:
3603 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3604 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3605 d.addCallback(self.failUnlessIsBarDotTxt)
3606 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3607 d.addCallback(self.failUnlessIsBarJSON)
3610 def test_POST_relink_file_replace(self):
3611 d = self.POST(self.public_url + "/foo", t="relink",
3612 from_name="bar.txt",
3613 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3614 d.addCallback(lambda res:
3615 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3616 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3617 d.addCallback(self.failUnlessIsBarDotTxt)
3618 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3619 d.addCallback(self.failUnlessIsBarJSON)
3622 def test_POST_relink_file_no_replace(self):
3623 d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3625 "There was already a child by that name, and you asked me to not replace it",
3626 self.POST, self.public_url + "/foo", t="relink",
3627 replace="false", from_name="bar.txt",
3628 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3629 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3630 d.addCallback(self.failUnlessIsBarDotTxt)
3631 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3632 d.addCallback(self.failUnlessIsBarJSON)
3633 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3634 d.addCallback(self.failUnlessIsSubBazDotTxt)
3637 def test_POST_relink_file_no_replace_explicitly_same_link(self):
3638 d = self.POST(self.public_url + "/foo", t="relink",
3639 replace="false", from_name="bar.txt",
3640 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3641 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3642 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3643 d.addCallback(self.failUnlessIsBarDotTxt)
3644 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3645 d.addCallback(self.failUnlessIsBarJSON)
3648 def test_POST_relink_file_replace_only_files(self):
3649 d = self.POST(self.public_url + "/foo", t="relink",
3650 replace="only-files", from_name="bar.txt",
3651 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3652 d.addCallback(lambda res:
3653 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3654 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3655 d.addCallback(self.failUnlessIsBarDotTxt)
3656 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3657 d.addCallback(self.failUnlessIsBarJSON)
3660 def test_POST_relink_file_replace_only_files_conflict(self):
3661 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3663 "There was already a child by that name, and you asked me to not replace it.",
3664 self.POST, self.public_url + "/foo", t="relink",
3665 replace="only-files", from_name="bar.txt",
3666 to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3667 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3668 d.addCallback(self.failUnlessIsBarDotTxt)
3669 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3670 d.addCallback(self.failUnlessIsBarJSON)
3673 def test_POST_relink_file_to_slash_fail(self):
3674 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3676 "to_name= may not contain a slash",
3677 self.POST, self.public_url + "/foo", t="relink",
3678 from_name="bar.txt",
3679 to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3680 d.addCallback(lambda res:
3681 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3682 d.addCallback(lambda res:
3683 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3684 d.addCallback(lambda ign:
3685 self.shouldFail2(error.Error,
3686 "test_POST_rename_file_slash_fail2",
3688 "from_name= may not contain a slash",
3689 self.POST, self.public_url + "/foo",
3691 from_name="nope/bar.txt",
3693 to_dir=self.public_root.get_uri() + "/foo/sub"))
3696 def test_POST_relink_file_explicitly_same_link(self):
3697 d = self.POST(self.public_url + "/foo", t="relink",
3698 from_name="bar.txt",
3699 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3700 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3701 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3702 d.addCallback(self.failUnlessIsBarDotTxt)
3703 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3704 d.addCallback(self.failUnlessIsBarJSON)
3707 def test_POST_relink_file_implicitly_same_link(self):
3708 d = self.POST(self.public_url + "/foo", t="relink",
3709 from_name="bar.txt")
3710 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3711 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3712 d.addCallback(self.failUnlessIsBarDotTxt)
3713 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3714 d.addCallback(self.failUnlessIsBarJSON)
3717 def test_POST_relink_file_same_dir(self):
3718 d = self.POST(self.public_url + "/foo", t="relink",
3719 from_name="bar.txt",
3720 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3721 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3722 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3723 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3724 d.addCallback(self.failUnlessIsBarDotTxt)
3725 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3726 d.addCallback(self.failUnlessIsBarJSON)
3729 def test_POST_relink_file_bad_replace(self):
3730 d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3731 "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3733 self.public_url + "/foo", t="relink",
3734 replace="boogabooga", from_name="bar.txt",
3735 to_dir=self.public_root.get_uri() + "/foo/sub")
3738 def test_POST_relink_file_multi_level(self):
3739 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3740 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3741 from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3742 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3743 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3744 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3745 d.addCallback(self.failUnlessIsBarDotTxt)
3746 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3747 d.addCallback(self.failUnlessIsBarJSON)
3750 def test_POST_relink_file_to_uri(self):
3751 d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3752 from_name="bar.txt", to_dir=self._sub_uri)
3753 d.addCallback(lambda res:
3754 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3755 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3756 d.addCallback(self.failUnlessIsBarDotTxt)
3757 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3758 d.addCallback(self.failUnlessIsBarJSON)
3761 def test_POST_relink_file_to_nonexistent_dir(self):
3762 d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3763 "404 Not Found", "No such child: nopechucktesta",
3764 self.POST, self.public_url + "/foo", t="relink",
3765 from_name="bar.txt",
3766 to_dir=self.public_root.get_uri() + "/nopechucktesta")
3769 def test_POST_relink_file_into_file(self):
3770 d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3771 "400 Bad Request", "to_dir is not a directory",
3772 self.POST, self.public_url + "/foo", t="relink",
3773 from_name="bar.txt",
3774 to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3775 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3776 d.addCallback(self.failUnlessIsBazDotTxt)
3777 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3778 d.addCallback(self.failUnlessIsBarDotTxt)
3779 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3780 d.addCallback(self.failUnlessIsBarJSON)
3783 def test_POST_relink_file_to_bad_uri(self):
3784 d = self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3785 "400 Bad Request", "to_dir is not a directory",
3786 self.POST, self.public_url + "/foo", t="relink",
3787 from_name="bar.txt",
3788 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3789 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3790 d.addCallback(self.failUnlessIsBarDotTxt)
3791 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3792 d.addCallback(self.failUnlessIsBarJSON)
3795 def test_POST_relink_dir(self):
3796 d = self.POST(self.public_url + "/foo", t="relink",
3797 from_name="bar.txt",
3798 to_dir=self.public_root.get_uri() + "/foo/empty")
3799 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3800 t="relink", from_name="empty",
3801 to_dir=self.public_root.get_uri() + "/foo/sub"))
3802 d.addCallback(lambda res:
3803 self.failIfNodeHasChild(self._foo_node, u"empty"))
3804 d.addCallback(lambda res:
3805 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3806 d.addCallback(lambda res:
3807 self._sub_node.get_child_at_path(u"empty"))
3808 d.addCallback(lambda node:
3809 self.failUnlessNodeHasChild(node, u"bar.txt"))
3810 d.addCallback(lambda res:
3811 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3812 d.addCallback(self.failUnlessIsBarDotTxt)
3815 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3816 """ If target is not None then the redirection has to go to target. If
3817 statuscode is not None then the redirection has to be accomplished with
3818 that HTTP status code."""
3819 if not isinstance(res, failure.Failure):
3820 to_where = (target is None) and "somewhere" or ("to " + target)
3821 self.fail("%s: we were expecting to get redirected %s, not get an"
3822 " actual page: %s" % (which, to_where, res))
3823 res.trap(error.PageRedirect)
3824 if statuscode is not None:
3825 self.failUnlessReallyEqual(res.value.status, statuscode,
3826 "%s: not a redirect" % which)
3827 if target is not None:
3828 # the PageRedirect does not seem to capture the uri= query arg
3829 # properly, so we can't check for it.
3830 realtarget = self.webish_url + target
3831 self.failUnlessReallyEqual(res.value.location, realtarget,
3832 "%s: wrong target" % which)
3833 return res.value.location
3835 def test_GET_URI_form(self):
3836 base = "/uri?uri=%s" % self._bar_txt_uri
3837 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3838 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3840 d.addBoth(self.shouldRedirect, targetbase)
3841 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3842 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3843 d.addCallback(lambda res: self.GET(base+"&t=json"))
3844 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3845 d.addCallback(self.log, "about to get file by uri")
3846 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3847 d.addCallback(self.failUnlessIsBarDotTxt)
3848 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3849 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3850 followRedirect=True))
3851 d.addCallback(self.failUnlessIsFooJSON)
3852 d.addCallback(self.log, "got dir by uri")
3856 def test_GET_URI_form_bad(self):
3857 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3858 "400 Bad Request", "GET /uri requires uri=",
3862 def test_GET_rename_form(self):
3863 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3864 followRedirect=True)
3866 self.failUnlessIn('name="when_done" value="."', res)
3867 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3868 self.failUnlessIn(FAVICON_MARKUP, res)
3869 d.addCallback(_check)
3872 def log(self, res, msg):
3873 #print "MSG: %s RES: %s" % (msg, res)
3877 def test_GET_URI_URL(self):
3878 base = "/uri/%s" % self._bar_txt_uri
3880 d.addCallback(self.failUnlessIsBarDotTxt)
3881 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3882 d.addCallback(self.failUnlessIsBarDotTxt)
3883 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3884 d.addCallback(self.failUnlessIsBarDotTxt)
3887 def test_GET_URI_URL_dir(self):
3888 base = "/uri/%s?t=json" % self._foo_uri
3890 d.addCallback(self.failUnlessIsFooJSON)
3893 def test_GET_URI_URL_missing(self):
3894 base = "/uri/%s" % self._bad_file_uri
3895 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3896 http.GONE, None, "NotEnoughSharesError",
3898 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3899 # here? we must arrange for a download to fail after target.open()
3900 # has been called, and then inspect the response to see that it is
3901 # shorter than we expected.
3904 def test_PUT_DIRURL_uri(self):
3905 d = self.s.create_dirnode()
3907 new_uri = dn.get_uri()
3908 # replace /foo with a new (empty) directory
3909 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3910 d.addCallback(lambda res:
3911 self.failUnlessReallyEqual(res.strip(), new_uri))
3912 d.addCallback(lambda res:
3913 self.failUnlessRWChildURIIs(self.public_root,
3917 d.addCallback(_made_dir)
3920 def test_PUT_DIRURL_uri_noreplace(self):
3921 d = self.s.create_dirnode()
3923 new_uri = dn.get_uri()
3924 # replace /foo with a new (empty) directory, but ask that
3925 # replace=false, so it should fail
3926 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3927 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3929 self.public_url + "/foo?t=uri&replace=false",
3931 d.addCallback(lambda res:
3932 self.failUnlessRWChildURIIs(self.public_root,
3936 d.addCallback(_made_dir)
3939 def test_PUT_DIRURL_bad_t(self):
3940 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3941 "400 Bad Request", "PUT to a directory",
3942 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3943 d.addCallback(lambda res:
3944 self.failUnlessRWChildURIIs(self.public_root,
3949 def test_PUT_NEWFILEURL_uri(self):
3950 contents, n, new_uri = self.makefile(8)
3951 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3952 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3953 d.addCallback(lambda res:
3954 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3958 def test_PUT_NEWFILEURL_mdmf(self):
3959 new_contents = self.NEWFILE_CONTENTS * 300000
3960 d = self.PUT(self.public_url + \
3961 "/foo/mdmf.txt?format=mdmf",
3963 d.addCallback(lambda ignored:
3964 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3965 def _got_json(json):
3966 data = simplejson.loads(json)
3968 self.failUnlessIn("format", data)
3969 self.failUnlessEqual(data["format"], "MDMF")
3970 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3971 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3972 d.addCallback(_got_json)
3975 def test_PUT_NEWFILEURL_sdmf(self):
3976 new_contents = self.NEWFILE_CONTENTS * 300000
3977 d = self.PUT(self.public_url + \
3978 "/foo/sdmf.txt?format=sdmf",
3980 d.addCallback(lambda ignored:
3981 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3982 def _got_json(json):
3983 data = simplejson.loads(json)
3985 self.failUnlessIn("format", data)
3986 self.failUnlessEqual(data["format"], "SDMF")
3987 d.addCallback(_got_json)
3990 def test_PUT_NEWFILEURL_bad_format(self):
3991 new_contents = self.NEWFILE_CONTENTS * 300000
3992 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3993 400, "Bad Request", "Unknown format: foo",
3994 self.PUT, self.public_url + \
3995 "/foo/foo.txt?format=foo",
3998 def test_PUT_NEWFILEURL_uri_replace(self):
3999 contents, n, new_uri = self.makefile(8)
4000 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4001 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4002 d.addCallback(lambda res:
4003 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4007 def test_PUT_NEWFILEURL_uri_no_replace(self):
4008 contents, n, new_uri = self.makefile(8)
4009 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4010 d.addBoth(self.shouldFail, error.Error,
4011 "PUT_NEWFILEURL_uri_no_replace",
4013 "There was already a child by that name, and you asked me "
4014 "to not replace it")
4017 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4018 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4019 d.addBoth(self.shouldFail, error.Error,
4020 "POST_put_uri_unknown_bad",
4022 "unknown cap in a write slot")
4025 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4026 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4027 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4028 u"put-future-ro.txt")
4031 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4032 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4033 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4034 u"put-future-imm.txt")
4037 def test_PUT_NEWFILE_URI(self):
4038 file_contents = "New file contents here\n"
4039 d = self.PUT("/uri", file_contents)
4041 assert isinstance(uri, str), uri
4042 self.failUnlessIn(uri, self.get_all_contents())
4043 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4045 return self.GET("/uri/%s" % uri)
4046 d.addCallback(_check)
4048 self.failUnlessReallyEqual(res, file_contents)
4049 d.addCallback(_check2)
4052 def test_PUT_NEWFILE_URI_not_mutable(self):
4053 file_contents = "New file contents here\n"
4054 d = self.PUT("/uri?mutable=false", file_contents)
4056 assert isinstance(uri, str), uri
4057 self.failUnlessIn(uri, self.get_all_contents())
4058 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4060 return self.GET("/uri/%s" % uri)
4061 d.addCallback(_check)
4063 self.failUnlessReallyEqual(res, file_contents)
4064 d.addCallback(_check2)
4067 def test_PUT_NEWFILE_URI_only_PUT(self):
4068 d = self.PUT("/uri?t=bogus", "")
4069 d.addBoth(self.shouldFail, error.Error,
4070 "PUT_NEWFILE_URI_only_PUT",
4072 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4075 def test_PUT_NEWFILE_URI_mutable(self):
4076 file_contents = "New file contents here\n"
4077 d = self.PUT("/uri?mutable=true", file_contents)
4078 def _check1(filecap):
4079 filecap = filecap.strip()
4080 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4081 self.filecap = filecap
4082 u = uri.WriteableSSKFileURI.init_from_string(filecap)
4083 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4084 n = self.s.create_node_from_uri(filecap)
4085 return n.download_best_version()
4086 d.addCallback(_check1)
4088 self.failUnlessReallyEqual(data, file_contents)
4089 return self.GET("/uri/%s" % urllib.quote(self.filecap))
4090 d.addCallback(_check2)
4092 self.failUnlessReallyEqual(res, file_contents)
4093 d.addCallback(_check3)
4096 def test_PUT_mkdir(self):
4097 d = self.PUT("/uri?t=mkdir", "")
4099 n = self.s.create_node_from_uri(uri.strip())
4100 d2 = self.failUnlessNodeKeysAre(n, [])
4101 d2.addCallback(lambda res:
4102 self.GET("/uri/%s?t=json" % uri))
4104 d.addCallback(_check)
4105 d.addCallback(self.failUnlessIsEmptyJSON)
4108 def test_PUT_mkdir_mdmf(self):
4109 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4111 u = uri.from_string(res)
4112 # Check that this is an MDMF writecap
4113 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4117 def test_PUT_mkdir_sdmf(self):
4118 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4120 u = uri.from_string(res)
4121 self.failUnlessIsInstance(u, uri.DirectoryURI)
4125 def test_PUT_mkdir_bad_format(self):
4126 return self.shouldHTTPError("PUT_mkdir_bad_format",
4127 400, "Bad Request", "Unknown format: foo",
4128 self.PUT, "/uri?t=mkdir&format=foo",
4131 def test_POST_check(self):
4132 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4134 # this returns a string form of the results, which are probably
4135 # None since we're using fake filenodes.
4136 # TODO: verify that the check actually happened, by changing
4137 # FakeCHKFileNode to count how many times .check() has been
4140 d.addCallback(_done)
4144 def test_PUT_update_at_offset(self):
4145 file_contents = "test file" * 100000 # about 900 KiB
4146 d = self.PUT("/uri?mutable=true", file_contents)
4148 self.filecap = filecap
4149 new_data = file_contents[:100]
4150 new = "replaced and so on"
4152 new_data += file_contents[len(new_data):]
4153 assert len(new_data) == len(file_contents)
4154 self.new_data = new_data
4155 d.addCallback(_then)
4156 d.addCallback(lambda ignored:
4157 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4158 "replaced and so on"))
4159 def _get_data(filecap):
4160 n = self.s.create_node_from_uri(filecap)
4161 return n.download_best_version()
4162 d.addCallback(_get_data)
4163 d.addCallback(lambda results:
4164 self.failUnlessEqual(results, self.new_data))
4165 # Now try appending things to the file
4166 d.addCallback(lambda ignored:
4167 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4169 d.addCallback(_get_data)
4170 d.addCallback(lambda results:
4171 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4172 # and try replacing the beginning of the file
4173 d.addCallback(lambda ignored:
4174 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4175 d.addCallback(_get_data)
4176 d.addCallback(lambda results:
4177 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4180 def test_PUT_update_at_invalid_offset(self):
4181 file_contents = "test file" * 100000 # about 900 KiB
4182 d = self.PUT("/uri?mutable=true", file_contents)
4184 self.filecap = filecap
4185 d.addCallback(_then)
4186 # Negative offsets should cause an error.
4187 d.addCallback(lambda ignored:
4188 self.shouldHTTPError("PUT_update_at_invalid_offset",
4192 "/uri/%s?offset=-1" % self.filecap,
4196 def test_PUT_update_at_offset_immutable(self):
4197 file_contents = "Test file" * 100000
4198 d = self.PUT("/uri", file_contents)
4200 self.filecap = filecap
4201 d.addCallback(_then)
4202 d.addCallback(lambda ignored:
4203 self.shouldHTTPError("PUT_update_at_offset_immutable",
4207 "/uri/%s?offset=50" % self.filecap,
4212 def test_bad_method(self):
4213 url = self.webish_url + self.public_url + "/foo/bar.txt"
4214 d = self.shouldHTTPError("bad_method",
4215 501, "Not Implemented",
4216 "I don't know how to treat a BOGUS request.",
4217 client.getPage, url, method="BOGUS")
4220 def test_short_url(self):
4221 url = self.webish_url + "/uri"
4222 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4223 "I don't know how to treat a DELETE request.",
4224 client.getPage, url, method="DELETE")
4227 def test_ophandle_bad(self):
4228 url = self.webish_url + "/operations/bogus?t=status"
4229 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4230 "unknown/expired handle 'bogus'",
4231 client.getPage, url)
4234 def test_ophandle_cancel(self):
4235 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4236 followRedirect=True)
4237 d.addCallback(lambda ignored:
4238 self.GET("/operations/128?t=status&output=JSON"))
4240 data = simplejson.loads(res)
4241 self.failUnless("finished" in data, res)
4242 monitor = self.ws.root.child_operations.handles["128"][0]
4243 d = self.POST("/operations/128?t=cancel&output=JSON")
4245 data = simplejson.loads(res)
4246 self.failUnless("finished" in data, res)
4247 # t=cancel causes the handle to be forgotten
4248 self.failUnless(monitor.is_cancelled())
4249 d.addCallback(_check2)
4251 d.addCallback(_check1)
4252 d.addCallback(lambda ignored:
4253 self.shouldHTTPError("ophandle_cancel",
4254 404, "404 Not Found",
4255 "unknown/expired handle '128'",
4257 "/operations/128?t=status&output=JSON"))
4260 def test_ophandle_retainfor(self):
4261 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4262 followRedirect=True)
4263 d.addCallback(lambda ignored:
4264 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4266 data = simplejson.loads(res)
4267 self.failUnless("finished" in data, res)
4268 d.addCallback(_check1)
4269 # the retain-for=0 will cause the handle to be expired very soon
4270 d.addCallback(lambda ign:
4271 self.clock.advance(2.0))
4272 d.addCallback(lambda ignored:
4273 self.shouldHTTPError("ophandle_retainfor",
4274 404, "404 Not Found",
4275 "unknown/expired handle '129'",
4277 "/operations/129?t=status&output=JSON"))
4280 def test_ophandle_release_after_complete(self):
4281 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4282 followRedirect=True)
4283 d.addCallback(self.wait_for_operation, "130")
4284 d.addCallback(lambda ignored:
4285 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4286 # the release-after-complete=true will cause the handle to be expired
4287 d.addCallback(lambda ignored:
4288 self.shouldHTTPError("ophandle_release_after_complete",
4289 404, "404 Not Found",
4290 "unknown/expired handle '130'",
4292 "/operations/130?t=status&output=JSON"))
4295 def test_uncollected_ophandle_expiration(self):
4296 # uncollected ophandles should expire after 4 days
4297 def _make_uncollected_ophandle(ophandle):
4298 d = self.POST(self.public_url +
4299 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4300 followRedirect=False)
4301 # When we start the operation, the webapi server will want
4302 # to redirect us to the page for the ophandle, so we get
4303 # confirmation that the operation has started. If the
4304 # manifest operation has finished by the time we get there,
4305 # following that redirect (by setting followRedirect=True
4306 # above) has the side effect of collecting the ophandle that
4307 # we've just created, which means that we can't use the
4308 # ophandle to test the uncollected timeout anymore. So,
4309 # instead, catch the 302 here and don't follow it.
4310 d.addBoth(self.should302, "uncollected_ophandle_creation")
4312 # Create an ophandle, don't collect it, then advance the clock by
4313 # 4 days - 1 second and make sure that the ophandle is still there.
4314 d = _make_uncollected_ophandle(131)
4315 d.addCallback(lambda ign:
4316 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4317 d.addCallback(lambda ign:
4318 self.GET("/operations/131?t=status&output=JSON"))
4320 data = simplejson.loads(res)
4321 self.failUnless("finished" in data, res)
4322 d.addCallback(_check1)
4323 # Create an ophandle, don't collect it, then try to collect it
4324 # after 4 days. It should be gone.
4325 d.addCallback(lambda ign:
4326 _make_uncollected_ophandle(132))
4327 d.addCallback(lambda ign:
4328 self.clock.advance(96*60*60))
4329 d.addCallback(lambda ign:
4330 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4331 404, "404 Not Found",
4332 "unknown/expired handle '132'",
4334 "/operations/132?t=status&output=JSON"))
4337 def test_collected_ophandle_expiration(self):
4338 # collected ophandles should expire after 1 day
4339 def _make_collected_ophandle(ophandle):
4340 d = self.POST(self.public_url +
4341 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4342 followRedirect=True)
4343 # By following the initial redirect, we collect the ophandle
4344 # we've just created.
4346 # Create a collected ophandle, then collect it after 23 hours
4347 # and 59 seconds to make sure that it is still there.
4348 d = _make_collected_ophandle(133)
4349 d.addCallback(lambda ign:
4350 self.clock.advance((24*60*60) - 1))
4351 d.addCallback(lambda ign:
4352 self.GET("/operations/133?t=status&output=JSON"))
4354 data = simplejson.loads(res)
4355 self.failUnless("finished" in data, res)
4356 d.addCallback(_check1)
4357 # Create another uncollected ophandle, then try to collect it
4358 # after 24 hours to make sure that it is gone.
4359 d.addCallback(lambda ign:
4360 _make_collected_ophandle(134))
4361 d.addCallback(lambda ign:
4362 self.clock.advance(24*60*60))
4363 d.addCallback(lambda ign:
4364 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4365 404, "404 Not Found",
4366 "unknown/expired handle '134'",
4368 "/operations/134?t=status&output=JSON"))
4371 def test_incident(self):
4372 d = self.POST("/report_incident", details="eek")
4374 self.failIfIn("<html>", res)
4375 self.failUnlessIn("Thank you for your report!", res)
4376 d.addCallback(_done)
4379 def test_static(self):
4380 webdir = os.path.join(self.staticdir, "subdir")
4381 fileutil.make_dirs(webdir)
4382 f = open(os.path.join(webdir, "hello.txt"), "wb")
4386 d = self.GET("/static/subdir/hello.txt")
4388 self.failUnlessReallyEqual(res, "hello")
4389 d.addCallback(_check)
4393 class IntroducerWeb(unittest.TestCase):
4398 d = defer.succeed(None)
4400 d.addCallback(lambda ign: self.node.stopService())
4401 d.addCallback(flushEventualQueue)
4404 def test_welcome(self):
4405 basedir = "web.IntroducerWeb.test_welcome"
4407 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4408 self.node = IntroducerNode(basedir)
4409 self.ws = self.node.getServiceNamed("webish")
4411 d = fireEventually(None)
4412 d.addCallback(lambda ign: self.node.startService())
4413 d.addCallback(lambda ign: self.node.when_tub_ready())
4415 d.addCallback(lambda ign: self.GET("/"))
4417 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4418 self.failUnlessIn(FAVICON_MARKUP, res)
4419 d.addCallback(_check)
4422 def GET(self, urlpath, followRedirect=False, return_response=False,
4424 # if return_response=True, this fires with (data, statuscode,
4425 # respheaders) instead of just data.
4426 assert not isinstance(urlpath, unicode)
4427 url = self.ws.getURL().rstrip('/') + urlpath
4428 factory = HTTPClientGETFactory(url, method="GET",
4429 followRedirect=followRedirect, **kwargs)
4430 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4431 d = factory.deferred
4432 def _got_data(data):
4433 return (data, factory.status, factory.response_headers)
4435 d.addCallback(_got_data)
4436 return factory.deferred
4439 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4440 def test_load_file(self):
4441 # This will raise an exception unless a well-formed XML file is found under that name.
4442 common.getxmlfile('directory.xhtml').load()
4444 def test_parse_replace_arg(self):
4445 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4446 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4447 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4449 self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4451 def test_abbreviate_time(self):
4452 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4453 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4454 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4455 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4456 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4457 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4459 def test_compute_rate(self):
4460 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4461 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4462 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4463 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4464 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4465 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4466 self.shouldFail(AssertionError, "test_compute_rate", "",
4467 common.compute_rate, -100, 10)
4468 self.shouldFail(AssertionError, "test_compute_rate", "",
4469 common.compute_rate, 100, -10)
4472 rate = common.compute_rate(10*1000*1000, 1)
4473 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4475 def test_abbreviate_rate(self):
4476 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4477 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4478 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4479 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4481 def test_abbreviate_size(self):
4482 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4483 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4484 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4485 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4486 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4488 def test_plural(self):
4490 return "%d second%s" % (s, status.plural(s))
4491 self.failUnlessReallyEqual(convert(0), "0 seconds")
4492 self.failUnlessReallyEqual(convert(1), "1 second")
4493 self.failUnlessReallyEqual(convert(2), "2 seconds")
4495 return "has share%s: %s" % (status.plural(s), ",".join(s))
4496 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4497 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4498 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4501 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4503 def CHECK(self, ign, which, args, clientnum=0):
4504 fileurl = self.fileurls[which]
4505 url = fileurl + "?" + args
4506 return self.GET(url, method="POST", clientnum=clientnum)
4508 def test_filecheck(self):
4509 self.basedir = "web/Grid/filecheck"
4511 c0 = self.g.clients[0]
4514 d = c0.upload(upload.Data(DATA, convergence=""))
4515 def _stash_uri(ur, which):
4516 self.uris[which] = ur.get_uri()
4517 d.addCallback(_stash_uri, "good")
4518 d.addCallback(lambda ign:
4519 c0.upload(upload.Data(DATA+"1", convergence="")))
4520 d.addCallback(_stash_uri, "sick")
4521 d.addCallback(lambda ign:
4522 c0.upload(upload.Data(DATA+"2", convergence="")))
4523 d.addCallback(_stash_uri, "dead")
4524 def _stash_mutable_uri(n, which):
4525 self.uris[which] = n.get_uri()
4526 assert isinstance(self.uris[which], str)
4527 d.addCallback(lambda ign:
4528 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4529 d.addCallback(_stash_mutable_uri, "corrupt")
4530 d.addCallback(lambda ign:
4531 c0.upload(upload.Data("literal", convergence="")))
4532 d.addCallback(_stash_uri, "small")
4533 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4534 d.addCallback(_stash_mutable_uri, "smalldir")
4536 def _compute_fileurls(ignored):
4538 for which in self.uris:
4539 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4540 d.addCallback(_compute_fileurls)
4542 def _clobber_shares(ignored):
4543 good_shares = self.find_uri_shares(self.uris["good"])
4544 self.failUnlessReallyEqual(len(good_shares), 10)
4545 sick_shares = self.find_uri_shares(self.uris["sick"])
4546 os.unlink(sick_shares[0][2])
4547 dead_shares = self.find_uri_shares(self.uris["dead"])
4548 for i in range(1, 10):
4549 os.unlink(dead_shares[i][2])
4550 c_shares = self.find_uri_shares(self.uris["corrupt"])
4551 cso = CorruptShareOptions()
4552 cso.stdout = StringIO()
4553 cso.parseOptions([c_shares[0][2]])
4555 d.addCallback(_clobber_shares)
4557 d.addCallback(self.CHECK, "good", "t=check")
4558 def _got_html_good(res):
4559 self.failUnlessIn("Healthy", res)
4560 self.failIfIn("Not Healthy", res)
4561 self.failUnlessIn(FAVICON_MARKUP, res)
4562 d.addCallback(_got_html_good)
4563 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4564 def _got_html_good_return_to(res):
4565 self.failUnlessIn("Healthy", res)
4566 self.failIfIn("Not Healthy", res)
4567 self.failUnlessIn('<a href="somewhere">Return to file', res)
4568 d.addCallback(_got_html_good_return_to)
4569 d.addCallback(self.CHECK, "good", "t=check&output=json")
4570 def _got_json_good(res):
4571 r = simplejson.loads(res)
4572 self.failUnlessEqual(r["summary"], "Healthy")
4573 self.failUnless(r["results"]["healthy"])
4574 self.failIf(r["results"]["needs-rebalancing"])
4575 self.failUnless(r["results"]["recoverable"])
4576 d.addCallback(_got_json_good)
4578 d.addCallback(self.CHECK, "small", "t=check")
4579 def _got_html_small(res):
4580 self.failUnlessIn("Literal files are always healthy", res)
4581 self.failIfIn("Not Healthy", res)
4582 d.addCallback(_got_html_small)
4583 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4584 def _got_html_small_return_to(res):
4585 self.failUnlessIn("Literal files are always healthy", res)
4586 self.failIfIn("Not Healthy", res)
4587 self.failUnlessIn('<a href="somewhere">Return to file', res)
4588 d.addCallback(_got_html_small_return_to)
4589 d.addCallback(self.CHECK, "small", "t=check&output=json")
4590 def _got_json_small(res):
4591 r = simplejson.loads(res)
4592 self.failUnlessEqual(r["storage-index"], "")
4593 self.failUnless(r["results"]["healthy"])
4594 d.addCallback(_got_json_small)
4596 d.addCallback(self.CHECK, "smalldir", "t=check")
4597 def _got_html_smalldir(res):
4598 self.failUnlessIn("Literal files are always healthy", res)
4599 self.failIfIn("Not Healthy", res)
4600 d.addCallback(_got_html_smalldir)
4601 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4602 def _got_json_smalldir(res):
4603 r = simplejson.loads(res)
4604 self.failUnlessEqual(r["storage-index"], "")
4605 self.failUnless(r["results"]["healthy"])
4606 d.addCallback(_got_json_smalldir)
4608 d.addCallback(self.CHECK, "sick", "t=check")
4609 def _got_html_sick(res):
4610 self.failUnlessIn("Not Healthy", res)
4611 d.addCallback(_got_html_sick)
4612 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4613 def _got_json_sick(res):
4614 r = simplejson.loads(res)
4615 self.failUnlessEqual(r["summary"],
4616 "Not Healthy: 9 shares (enc 3-of-10)")
4617 self.failIf(r["results"]["healthy"])
4618 self.failIf(r["results"]["needs-rebalancing"])
4619 self.failUnless(r["results"]["recoverable"])
4620 d.addCallback(_got_json_sick)
4622 d.addCallback(self.CHECK, "dead", "t=check")
4623 def _got_html_dead(res):
4624 self.failUnlessIn("Not Healthy", res)
4625 d.addCallback(_got_html_dead)
4626 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4627 def _got_json_dead(res):
4628 r = simplejson.loads(res)
4629 self.failUnlessEqual(r["summary"],
4630 "Not Healthy: 1 shares (enc 3-of-10)")
4631 self.failIf(r["results"]["healthy"])
4632 self.failIf(r["results"]["needs-rebalancing"])
4633 self.failIf(r["results"]["recoverable"])
4634 d.addCallback(_got_json_dead)
4636 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4637 def _got_html_corrupt(res):
4638 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4639 d.addCallback(_got_html_corrupt)
4640 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4641 def _got_json_corrupt(res):
4642 r = simplejson.loads(res)
4643 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4644 self.failIf(r["results"]["healthy"])
4645 self.failUnless(r["results"]["recoverable"])
4646 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4647 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4648 d.addCallback(_got_json_corrupt)
4650 d.addErrback(self.explain_web_error)
4653 def test_repair_html(self):
4654 self.basedir = "web/Grid/repair_html"
4656 c0 = self.g.clients[0]
4659 d = c0.upload(upload.Data(DATA, convergence=""))
4660 def _stash_uri(ur, which):
4661 self.uris[which] = ur.get_uri()
4662 d.addCallback(_stash_uri, "good")
4663 d.addCallback(lambda ign:
4664 c0.upload(upload.Data(DATA+"1", convergence="")))
4665 d.addCallback(_stash_uri, "sick")
4666 d.addCallback(lambda ign:
4667 c0.upload(upload.Data(DATA+"2", convergence="")))
4668 d.addCallback(_stash_uri, "dead")
4669 def _stash_mutable_uri(n, which):
4670 self.uris[which] = n.get_uri()
4671 assert isinstance(self.uris[which], str)
4672 d.addCallback(lambda ign:
4673 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4674 d.addCallback(_stash_mutable_uri, "corrupt")
4676 def _compute_fileurls(ignored):
4678 for which in self.uris:
4679 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4680 d.addCallback(_compute_fileurls)
4682 def _clobber_shares(ignored):
4683 good_shares = self.find_uri_shares(self.uris["good"])
4684 self.failUnlessReallyEqual(len(good_shares), 10)
4685 sick_shares = self.find_uri_shares(self.uris["sick"])
4686 os.unlink(sick_shares[0][2])
4687 dead_shares = self.find_uri_shares(self.uris["dead"])
4688 for i in range(1, 10):
4689 os.unlink(dead_shares[i][2])
4690 c_shares = self.find_uri_shares(self.uris["corrupt"])
4691 cso = CorruptShareOptions()
4692 cso.stdout = StringIO()
4693 cso.parseOptions([c_shares[0][2]])
4695 d.addCallback(_clobber_shares)
4697 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4698 def _got_html_good(res):
4699 self.failUnlessIn("Healthy", res)
4700 self.failIfIn("Not Healthy", res)
4701 self.failUnlessIn("No repair necessary", res)
4702 self.failUnlessIn(FAVICON_MARKUP, res)
4703 d.addCallback(_got_html_good)
4705 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4706 def _got_html_sick(res):
4707 self.failUnlessIn("Healthy : healthy", res)
4708 self.failIfIn("Not Healthy", res)
4709 self.failUnlessIn("Repair successful", res)
4710 d.addCallback(_got_html_sick)
4712 # repair of a dead file will fail, of course, but it isn't yet
4713 # clear how this should be reported. Right now it shows up as
4716 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4717 #def _got_html_dead(res):
4719 # self.failUnlessIn("Healthy : healthy", res)
4720 # self.failIfIn("Not Healthy", res)
4721 # self.failUnlessIn("No repair necessary", res)
4722 #d.addCallback(_got_html_dead)
4724 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4725 def _got_html_corrupt(res):
4726 self.failUnlessIn("Healthy : Healthy", res)
4727 self.failIfIn("Not Healthy", res)
4728 self.failUnlessIn("Repair successful", res)
4729 d.addCallback(_got_html_corrupt)
4731 d.addErrback(self.explain_web_error)
4734 def test_repair_json(self):
4735 self.basedir = "web/Grid/repair_json"
4737 c0 = self.g.clients[0]
4740 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4741 def _stash_uri(ur, which):
4742 self.uris[which] = ur.get_uri()
4743 d.addCallback(_stash_uri, "sick")
4745 def _compute_fileurls(ignored):
4747 for which in self.uris:
4748 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4749 d.addCallback(_compute_fileurls)
4751 def _clobber_shares(ignored):
4752 sick_shares = self.find_uri_shares(self.uris["sick"])
4753 os.unlink(sick_shares[0][2])
4754 d.addCallback(_clobber_shares)
4756 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4757 def _got_json_sick(res):
4758 r = simplejson.loads(res)
4759 self.failUnlessReallyEqual(r["repair-attempted"], True)
4760 self.failUnlessReallyEqual(r["repair-successful"], True)
4761 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4762 "Not Healthy: 9 shares (enc 3-of-10)")
4763 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4764 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4765 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4766 d.addCallback(_got_json_sick)
4768 d.addErrback(self.explain_web_error)
4771 def test_unknown(self, immutable=False):
4772 self.basedir = "web/Grid/unknown"
4774 self.basedir = "web/Grid/unknown-immutable"
4777 c0 = self.g.clients[0]
4781 # the future cap format may contain slashes, which must be tolerated
4782 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4786 name = u"future-imm"
4787 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4788 d = c0.create_immutable_dirnode({name: (future_node, {})})
4791 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4792 d = c0.create_dirnode()
4794 def _stash_root_and_create_file(n):
4796 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4797 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4799 return self.rootnode.set_node(name, future_node)
4800 d.addCallback(_stash_root_and_create_file)
4802 # make sure directory listing tolerates unknown nodes
4803 d.addCallback(lambda ign: self.GET(self.rooturl))
4804 def _check_directory_html(res, expected_type_suffix):
4805 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4806 '<td>%s</td>' % (expected_type_suffix, str(name)),
4808 self.failUnless(re.search(pattern, res), res)
4809 # find the More Info link for name, should be relative
4810 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4811 info_url = mo.group(1)
4812 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4814 d.addCallback(_check_directory_html, "-IMM")
4816 d.addCallback(_check_directory_html, "")
4818 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4819 def _check_directory_json(res, expect_rw_uri):
4820 data = simplejson.loads(res)
4821 self.failUnlessEqual(data[0], "dirnode")
4822 f = data[1]["children"][name]
4823 self.failUnlessEqual(f[0], "unknown")
4825 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4827 self.failIfIn("rw_uri", f[1])
4829 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4831 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4832 self.failUnlessIn("metadata", f[1])
4833 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4835 def _check_info(res, expect_rw_uri, expect_ro_uri):
4836 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4838 self.failUnlessIn(unknown_rwcap, res)
4841 self.failUnlessIn(unknown_immcap, res)
4843 self.failUnlessIn(unknown_rocap, res)
4845 self.failIfIn(unknown_rocap, res)
4846 self.failIfIn("Raw data as", res)
4847 self.failIfIn("Directory writecap", res)
4848 self.failIfIn("Checker Operations", res)
4849 self.failIfIn("Mutable File Operations", res)
4850 self.failIfIn("Directory Operations", res)
4852 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4853 # why they fail. Possibly related to ticket #922.
4855 d.addCallback(lambda ign: self.GET(expected_info_url))
4856 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4857 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4858 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4860 def _check_json(res, expect_rw_uri):
4861 data = simplejson.loads(res)
4862 self.failUnlessEqual(data[0], "unknown")
4864 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4866 self.failIfIn("rw_uri", data[1])
4869 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4870 self.failUnlessReallyEqual(data[1]["mutable"], False)
4872 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4873 self.failUnlessReallyEqual(data[1]["mutable"], True)
4875 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4876 self.failIfIn("mutable", data[1])
4878 # TODO: check metadata contents
4879 self.failUnlessIn("metadata", data[1])
4881 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4882 d.addCallback(_check_json, expect_rw_uri=not immutable)
4884 # and make sure that a read-only version of the directory can be
4885 # rendered too. This version will not have unknown_rwcap, whether
4886 # or not future_node was immutable.
4887 d.addCallback(lambda ign: self.GET(self.rourl))
4889 d.addCallback(_check_directory_html, "-IMM")
4891 d.addCallback(_check_directory_html, "-RO")
4893 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4894 d.addCallback(_check_directory_json, expect_rw_uri=False)
4896 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4897 d.addCallback(_check_json, expect_rw_uri=False)
4899 # TODO: check that getting t=info from the Info link in the ro directory
4900 # works, and does not include the writecap URI.
4903 def test_immutable_unknown(self):
4904 return self.test_unknown(immutable=True)
4906 def test_mutant_dirnodes_are_omitted(self):
4907 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4910 c = self.g.clients[0]
4915 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4916 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4917 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4919 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4920 # test the dirnode and web layers separately.
4922 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4923 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4924 # When the directory is read, the mutants should be silently disposed of, leaving
4925 # their lonely sibling.
4926 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4927 # because immutable directories don't have a writecap and therefore that field
4928 # isn't (and can't be) decrypted.
4929 # TODO: The field still exists in the netstring. Technically we should check what
4930 # happens if something is put there (_unpack_contents should raise ValueError),
4931 # but that can wait.
4933 lonely_child = nm.create_from_cap(lonely_uri)
4934 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4935 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4937 def _by_hook_or_by_crook():
4939 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4940 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4942 mutant_write_in_ro_child.get_write_uri = lambda: None
4943 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4945 kids = {u"lonely": (lonely_child, {}),
4946 u"ro": (mutant_ro_child, {}),
4947 u"write-in-ro": (mutant_write_in_ro_child, {}),
4949 d = c.create_immutable_dirnode(kids)
4952 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4953 self.failIf(dn.is_mutable())
4954 self.failUnless(dn.is_readonly())
4955 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4956 self.failIf(hasattr(dn._node, 'get_writekey'))
4958 self.failUnlessIn("RO-IMM", rep)
4960 self.failUnlessIn("CHK", cap.to_string())
4963 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4964 return download_to_data(dn._node)
4965 d.addCallback(_created)
4967 def _check_data(data):
4968 # Decode the netstring representation of the directory to check that all children
4969 # are present. This is a bit of an abstraction violation, but there's not really
4970 # any other way to do it given that the real DirectoryNode._unpack_contents would
4971 # strip the mutant children out (which is what we're trying to test, later).
4974 while position < len(data):
4975 entries, position = split_netstring(data, 1, position)
4977 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4978 name = name_utf8.decode("utf-8")
4979 self.failUnlessEqual(rwcapdata, "")
4980 self.failUnlessIn(name, kids)
4981 (expected_child, ign) = kids[name]
4982 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4985 self.failUnlessReallyEqual(numkids, 3)
4986 return self.rootnode.list()
4987 d.addCallback(_check_data)
4989 # Now when we use the real directory listing code, the mutants should be absent.
4990 def _check_kids(children):
4991 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4992 lonely_node, lonely_metadata = children[u"lonely"]
4994 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4995 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4996 d.addCallback(_check_kids)
4998 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4999 d.addCallback(lambda n: n.list())
5000 d.addCallback(_check_kids) # again with dirnode recreated from cap
5002 # Make sure the lonely child can be listed in HTML...
5003 d.addCallback(lambda ign: self.GET(self.rooturl))
5004 def _check_html(res):
5005 self.failIfIn("URI:SSK", res)
5006 get_lonely = "".join([r'<td>FILE</td>',
5008 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5010 r'\s+<td align="right">%d</td>' % len("one"),
5012 self.failUnless(re.search(get_lonely, res), res)
5014 # find the More Info link for name, should be relative
5015 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5016 info_url = mo.group(1)
5017 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5018 d.addCallback(_check_html)
5021 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5022 def _check_json(res):
5023 data = simplejson.loads(res)
5024 self.failUnlessEqual(data[0], "dirnode")
5025 listed_children = data[1]["children"]
5026 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5027 ll_type, ll_data = listed_children[u"lonely"]
5028 self.failUnlessEqual(ll_type, "filenode")
5029 self.failIfIn("rw_uri", ll_data)
5030 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5031 d.addCallback(_check_json)
5034 def test_deep_check(self):
5035 self.basedir = "web/Grid/deep_check"
5037 c0 = self.g.clients[0]
5041 d = c0.create_dirnode()
5042 def _stash_root_and_create_file(n):
5044 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5045 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5046 d.addCallback(_stash_root_and_create_file)
5047 def _stash_uri(fn, which):
5048 self.uris[which] = fn.get_uri()
5050 d.addCallback(_stash_uri, "good")
5051 d.addCallback(lambda ign:
5052 self.rootnode.add_file(u"small",
5053 upload.Data("literal",
5055 d.addCallback(_stash_uri, "small")
5056 d.addCallback(lambda ign:
5057 self.rootnode.add_file(u"sick",
5058 upload.Data(DATA+"1",
5060 d.addCallback(_stash_uri, "sick")
5062 # this tests that deep-check and stream-manifest will ignore
5063 # UnknownNode instances. Hopefully this will also cover deep-stats.
5064 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5065 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5067 def _clobber_shares(ignored):
5068 self.delete_shares_numbered(self.uris["sick"], [0,1])
5069 d.addCallback(_clobber_shares)
5077 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5080 units = [simplejson.loads(line)
5081 for line in res.splitlines()
5084 print "response is:", res
5085 print "undecodeable line was '%s'" % line
5087 self.failUnlessReallyEqual(len(units), 5+1)
5088 # should be parent-first
5090 self.failUnlessEqual(u0["path"], [])
5091 self.failUnlessEqual(u0["type"], "directory")
5092 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5093 u0cr = u0["check-results"]
5094 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5096 ugood = [u for u in units
5097 if u["type"] == "file" and u["path"] == [u"good"]][0]
5098 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5099 ugoodcr = ugood["check-results"]
5100 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5103 self.failUnlessEqual(stats["type"], "stats")
5105 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5106 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5107 self.failUnlessReallyEqual(s["count-directories"], 1)
5108 self.failUnlessReallyEqual(s["count-unknown"], 1)
5109 d.addCallback(_done)
5111 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5112 def _check_manifest(res):
5113 self.failUnless(res.endswith("\n"))
5114 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5115 self.failUnlessReallyEqual(len(units), 5+1)
5116 self.failUnlessEqual(units[-1]["type"], "stats")
5118 self.failUnlessEqual(first["path"], [])
5119 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5120 self.failUnlessEqual(first["type"], "directory")
5121 stats = units[-1]["stats"]
5122 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5123 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5124 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5125 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5126 self.failUnlessReallyEqual(stats["count-unknown"], 1)
5127 d.addCallback(_check_manifest)
5129 # now add root/subdir and root/subdir/grandchild, then make subdir
5130 # unrecoverable, then see what happens
5132 d.addCallback(lambda ign:
5133 self.rootnode.create_subdirectory(u"subdir"))
5134 d.addCallback(_stash_uri, "subdir")
5135 d.addCallback(lambda subdir_node:
5136 subdir_node.add_file(u"grandchild",
5137 upload.Data(DATA+"2",
5139 d.addCallback(_stash_uri, "grandchild")
5141 d.addCallback(lambda ign:
5142 self.delete_shares_numbered(self.uris["subdir"],
5150 # root/subdir [unrecoverable]
5151 # root/subdir/grandchild
5153 # how should a streaming-JSON API indicate fatal error?
5154 # answer: emit ERROR: instead of a JSON string
5156 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5157 def _check_broken_manifest(res):
5158 lines = res.splitlines()
5160 for (i,line) in enumerate(lines)
5161 if line.startswith("ERROR:")]
5163 self.fail("no ERROR: in output: %s" % (res,))
5164 first_error = error_lines[0]
5165 error_line = lines[first_error]
5166 error_msg = lines[first_error+1:]
5167 error_msg_s = "\n".join(error_msg) + "\n"
5168 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5170 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5171 units = [simplejson.loads(line) for line in lines[:first_error]]
5172 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5173 last_unit = units[-1]
5174 self.failUnlessEqual(last_unit["path"], ["subdir"])
5175 d.addCallback(_check_broken_manifest)
5177 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5178 def _check_broken_deepcheck(res):
5179 lines = res.splitlines()
5181 for (i,line) in enumerate(lines)
5182 if line.startswith("ERROR:")]
5184 self.fail("no ERROR: in output: %s" % (res,))
5185 first_error = error_lines[0]
5186 error_line = lines[first_error]
5187 error_msg = lines[first_error+1:]
5188 error_msg_s = "\n".join(error_msg) + "\n"
5189 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5191 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5192 units = [simplejson.loads(line) for line in lines[:first_error]]
5193 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5194 last_unit = units[-1]
5195 self.failUnlessEqual(last_unit["path"], ["subdir"])
5196 r = last_unit["check-results"]["results"]
5197 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5198 self.failUnlessReallyEqual(r["count-shares-good"], 1)
5199 self.failUnlessReallyEqual(r["recoverable"], False)
5200 d.addCallback(_check_broken_deepcheck)
5202 d.addErrback(self.explain_web_error)
5205 def test_deep_check_and_repair(self):
5206 self.basedir = "web/Grid/deep_check_and_repair"
5208 c0 = self.g.clients[0]
5212 d = c0.create_dirnode()
5213 def _stash_root_and_create_file(n):
5215 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5216 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5217 d.addCallback(_stash_root_and_create_file)
5218 def _stash_uri(fn, which):
5219 self.uris[which] = fn.get_uri()
5220 d.addCallback(_stash_uri, "good")
5221 d.addCallback(lambda ign:
5222 self.rootnode.add_file(u"small",
5223 upload.Data("literal",
5225 d.addCallback(_stash_uri, "small")
5226 d.addCallback(lambda ign:
5227 self.rootnode.add_file(u"sick",
5228 upload.Data(DATA+"1",
5230 d.addCallback(_stash_uri, "sick")
5231 #d.addCallback(lambda ign:
5232 # self.rootnode.add_file(u"dead",
5233 # upload.Data(DATA+"2",
5235 #d.addCallback(_stash_uri, "dead")
5237 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5238 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5239 #d.addCallback(_stash_uri, "corrupt")
5241 def _clobber_shares(ignored):
5242 good_shares = self.find_uri_shares(self.uris["good"])
5243 self.failUnlessReallyEqual(len(good_shares), 10)
5244 sick_shares = self.find_uri_shares(self.uris["sick"])
5245 os.unlink(sick_shares[0][2])
5246 #dead_shares = self.find_uri_shares(self.uris["dead"])
5247 #for i in range(1, 10):
5248 # os.unlink(dead_shares[i][2])
5250 #c_shares = self.find_uri_shares(self.uris["corrupt"])
5251 #cso = CorruptShareOptions()
5252 #cso.stdout = StringIO()
5253 #cso.parseOptions([c_shares[0][2]])
5255 d.addCallback(_clobber_shares)
5258 # root/good CHK, 10 shares
5260 # root/sick CHK, 9 shares
5262 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5264 units = [simplejson.loads(line)
5265 for line in res.splitlines()
5267 self.failUnlessReallyEqual(len(units), 4+1)
5268 # should be parent-first
5270 self.failUnlessEqual(u0["path"], [])
5271 self.failUnlessEqual(u0["type"], "directory")
5272 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5273 u0crr = u0["check-and-repair-results"]
5274 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5275 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5277 ugood = [u for u in units
5278 if u["type"] == "file" and u["path"] == [u"good"]][0]
5279 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5280 ugoodcrr = ugood["check-and-repair-results"]
5281 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5282 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5284 usick = [u for u in units
5285 if u["type"] == "file" and u["path"] == [u"sick"]][0]
5286 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5287 usickcrr = usick["check-and-repair-results"]
5288 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5289 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5290 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5291 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5294 self.failUnlessEqual(stats["type"], "stats")
5296 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5297 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5298 self.failUnlessReallyEqual(s["count-directories"], 1)
5299 d.addCallback(_done)
5301 d.addErrback(self.explain_web_error)
5304 def _count_leases(self, ignored, which):
5305 u = self.uris[which]
5306 shares = self.find_uri_shares(u)
5308 for shnum, serverid, fn in shares:
5309 sf = get_share_file(fn)
5310 num_leases = len(list(sf.get_leases()))
5311 lease_counts.append( (fn, num_leases) )
5314 def _assert_leasecount(self, lease_counts, expected):
5315 for (fn, num_leases) in lease_counts:
5316 if num_leases != expected:
5317 self.fail("expected %d leases, have %d, on %s" %
5318 (expected, num_leases, fn))
5320 def test_add_lease(self):
5321 self.basedir = "web/Grid/add_lease"
5322 self.set_up_grid(num_clients=2)
5323 c0 = self.g.clients[0]
5326 d = c0.upload(upload.Data(DATA, convergence=""))
5327 def _stash_uri(ur, which):
5328 self.uris[which] = ur.get_uri()
5329 d.addCallback(_stash_uri, "one")
5330 d.addCallback(lambda ign:
5331 c0.upload(upload.Data(DATA+"1", convergence="")))
5332 d.addCallback(_stash_uri, "two")
5333 def _stash_mutable_uri(n, which):
5334 self.uris[which] = n.get_uri()
5335 assert isinstance(self.uris[which], str)
5336 d.addCallback(lambda ign:
5337 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5338 d.addCallback(_stash_mutable_uri, "mutable")
5340 def _compute_fileurls(ignored):
5342 for which in self.uris:
5343 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5344 d.addCallback(_compute_fileurls)
5346 d.addCallback(self._count_leases, "one")
5347 d.addCallback(self._assert_leasecount, 1)
5348 d.addCallback(self._count_leases, "two")
5349 d.addCallback(self._assert_leasecount, 1)
5350 d.addCallback(self._count_leases, "mutable")
5351 d.addCallback(self._assert_leasecount, 1)
5353 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5354 def _got_html_good(res):
5355 self.failUnlessIn("Healthy", res)
5356 self.failIfIn("Not Healthy", res)
5357 d.addCallback(_got_html_good)
5359 d.addCallback(self._count_leases, "one")
5360 d.addCallback(self._assert_leasecount, 1)
5361 d.addCallback(self._count_leases, "two")
5362 d.addCallback(self._assert_leasecount, 1)
5363 d.addCallback(self._count_leases, "mutable")
5364 d.addCallback(self._assert_leasecount, 1)
5366 # this CHECK uses the original client, which uses the same
5367 # lease-secrets, so it will just renew the original lease
5368 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5369 d.addCallback(_got_html_good)
5371 d.addCallback(self._count_leases, "one")
5372 d.addCallback(self._assert_leasecount, 1)
5373 d.addCallback(self._count_leases, "two")
5374 d.addCallback(self._assert_leasecount, 1)
5375 d.addCallback(self._count_leases, "mutable")
5376 d.addCallback(self._assert_leasecount, 1)
5378 # this CHECK uses an alternate client, which adds a second lease
5379 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5380 d.addCallback(_got_html_good)
5382 d.addCallback(self._count_leases, "one")
5383 d.addCallback(self._assert_leasecount, 2)
5384 d.addCallback(self._count_leases, "two")
5385 d.addCallback(self._assert_leasecount, 1)
5386 d.addCallback(self._count_leases, "mutable")
5387 d.addCallback(self._assert_leasecount, 1)
5389 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5390 d.addCallback(_got_html_good)
5392 d.addCallback(self._count_leases, "one")
5393 d.addCallback(self._assert_leasecount, 2)
5394 d.addCallback(self._count_leases, "two")
5395 d.addCallback(self._assert_leasecount, 1)
5396 d.addCallback(self._count_leases, "mutable")
5397 d.addCallback(self._assert_leasecount, 1)
5399 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5401 d.addCallback(_got_html_good)
5403 d.addCallback(self._count_leases, "one")
5404 d.addCallback(self._assert_leasecount, 2)
5405 d.addCallback(self._count_leases, "two")
5406 d.addCallback(self._assert_leasecount, 1)
5407 d.addCallback(self._count_leases, "mutable")
5408 d.addCallback(self._assert_leasecount, 2)
5410 d.addErrback(self.explain_web_error)
5413 def test_deep_add_lease(self):
5414 self.basedir = "web/Grid/deep_add_lease"
5415 self.set_up_grid(num_clients=2)
5416 c0 = self.g.clients[0]
5420 d = c0.create_dirnode()
5421 def _stash_root_and_create_file(n):
5423 self.uris["root"] = n.get_uri()
5424 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5425 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5426 d.addCallback(_stash_root_and_create_file)
5427 def _stash_uri(fn, which):
5428 self.uris[which] = fn.get_uri()
5429 d.addCallback(_stash_uri, "one")
5430 d.addCallback(lambda ign:
5431 self.rootnode.add_file(u"small",
5432 upload.Data("literal",
5434 d.addCallback(_stash_uri, "small")
5436 d.addCallback(lambda ign:
5437 c0.create_mutable_file(publish.MutableData("mutable")))
5438 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5439 d.addCallback(_stash_uri, "mutable")
5441 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5443 units = [simplejson.loads(line)
5444 for line in res.splitlines()
5446 # root, one, small, mutable, stats
5447 self.failUnlessReallyEqual(len(units), 4+1)
5448 d.addCallback(_done)
5450 d.addCallback(self._count_leases, "root")
5451 d.addCallback(self._assert_leasecount, 1)
5452 d.addCallback(self._count_leases, "one")
5453 d.addCallback(self._assert_leasecount, 1)
5454 d.addCallback(self._count_leases, "mutable")
5455 d.addCallback(self._assert_leasecount, 1)
5457 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5458 d.addCallback(_done)
5460 d.addCallback(self._count_leases, "root")
5461 d.addCallback(self._assert_leasecount, 1)
5462 d.addCallback(self._count_leases, "one")
5463 d.addCallback(self._assert_leasecount, 1)
5464 d.addCallback(self._count_leases, "mutable")
5465 d.addCallback(self._assert_leasecount, 1)
5467 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5469 d.addCallback(_done)
5471 d.addCallback(self._count_leases, "root")
5472 d.addCallback(self._assert_leasecount, 2)
5473 d.addCallback(self._count_leases, "one")
5474 d.addCallback(self._assert_leasecount, 2)
5475 d.addCallback(self._count_leases, "mutable")
5476 d.addCallback(self._assert_leasecount, 2)
5478 d.addErrback(self.explain_web_error)
5482 def test_exceptions(self):
5483 self.basedir = "web/Grid/exceptions"
5484 self.set_up_grid(num_clients=1, num_servers=2)
5485 c0 = self.g.clients[0]
5486 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5489 d = c0.create_dirnode()
5491 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5492 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5494 d.addCallback(_stash_root)
5495 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5497 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5498 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5500 u = uri.from_string(ur.get_uri())
5501 u.key = testutil.flip_bit(u.key, 0)
5502 baduri = u.to_string()
5503 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5504 d.addCallback(_stash_bad)
5505 d.addCallback(lambda ign: c0.create_dirnode())
5506 def _mangle_dirnode_1share(n):
5508 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5509 self.fileurls["dir-1share-json"] = url + "?t=json"
5510 self.delete_shares_numbered(u, range(1,10))
5511 d.addCallback(_mangle_dirnode_1share)
5512 d.addCallback(lambda ign: c0.create_dirnode())
5513 def _mangle_dirnode_0share(n):
5515 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5516 self.fileurls["dir-0share-json"] = url + "?t=json"
5517 self.delete_shares_numbered(u, range(0,10))
5518 d.addCallback(_mangle_dirnode_0share)
5520 # NotEnoughSharesError should be reported sensibly, with a
5521 # text/plain explanation of the problem, and perhaps some
5522 # information on which shares *could* be found.
5524 d.addCallback(lambda ignored:
5525 self.shouldHTTPError("GET unrecoverable",
5526 410, "Gone", "NoSharesError",
5527 self.GET, self.fileurls["0shares"]))
5528 def _check_zero_shares(body):
5529 self.failIfIn("<html>", body)
5530 body = " ".join(body.strip().split())
5531 exp = ("NoSharesError: no shares could be found. "
5532 "Zero shares usually indicates a corrupt URI, or that "
5533 "no servers were connected, but it might also indicate "
5534 "severe corruption. You should perform a filecheck on "
5535 "this object to learn more. The full error message is: "
5536 "no shares (need 3). Last failure: None")
5537 self.failUnlessReallyEqual(exp, body)
5538 d.addCallback(_check_zero_shares)
5541 d.addCallback(lambda ignored:
5542 self.shouldHTTPError("GET 1share",
5543 410, "Gone", "NotEnoughSharesError",
5544 self.GET, self.fileurls["1share"]))
5545 def _check_one_share(body):
5546 self.failIfIn("<html>", body)
5547 body = " ".join(body.strip().split())
5548 msgbase = ("NotEnoughSharesError: This indicates that some "
5549 "servers were unavailable, or that shares have been "
5550 "lost to server departure, hard drive failure, or disk "
5551 "corruption. You should perform a filecheck on "
5552 "this object to learn more. The full error message is:"
5554 msg1 = msgbase + (" ran out of shares:"
5557 " overdue= unused= need 3. Last failure: None")
5558 msg2 = msgbase + (" ran out of shares:"
5560 " pending=Share(sh0-on-xgru5)"
5561 " overdue= unused= need 3. Last failure: None")
5562 self.failUnless(body == msg1 or body == msg2, body)
5563 d.addCallback(_check_one_share)
5565 d.addCallback(lambda ignored:
5566 self.shouldHTTPError("GET imaginary",
5567 404, "Not Found", None,
5568 self.GET, self.fileurls["imaginary"]))
5569 def _missing_child(body):
5570 self.failUnlessIn("No such child: imaginary", body)
5571 d.addCallback(_missing_child)
5573 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5574 def _check_0shares_dir_html(body):
5575 self.failUnlessIn("<html>", body)
5576 # we should see the regular page, but without the child table or
5578 body = " ".join(body.strip().split())
5579 self.failUnlessIn('href="?t=info">More info on this directory',
5581 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5582 "could not be retrieved, because there were insufficient "
5583 "good shares. This might indicate that no servers were "
5584 "connected, insufficient servers were connected, the URI "
5585 "was corrupt, or that shares have been lost due to server "
5586 "departure, hard drive failure, or disk corruption. You "
5587 "should perform a filecheck on this object to learn more.")
5588 self.failUnlessIn(exp, body)
5589 self.failUnlessIn("No upload forms: directory is unreadable", body)
5590 d.addCallback(_check_0shares_dir_html)
5592 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5593 def _check_1shares_dir_html(body):
5594 # at some point, we'll split UnrecoverableFileError into 0-shares
5595 # and some-shares like we did for immutable files (since there
5596 # are different sorts of advice to offer in each case). For now,
5597 # they present the same way.
5598 self.failUnlessIn("<html>", body)
5599 body = " ".join(body.strip().split())
5600 self.failUnlessIn('href="?t=info">More info on this directory',
5602 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5603 "could not be retrieved, because there were insufficient "
5604 "good shares. This might indicate that no servers were "
5605 "connected, insufficient servers were connected, the URI "
5606 "was corrupt, or that shares have been lost due to server "
5607 "departure, hard drive failure, or disk corruption. You "
5608 "should perform a filecheck on this object to learn more.")
5609 self.failUnlessIn(exp, body)
5610 self.failUnlessIn("No upload forms: directory is unreadable", body)
5611 d.addCallback(_check_1shares_dir_html)
5613 d.addCallback(lambda ignored:
5614 self.shouldHTTPError("GET dir-0share-json",
5615 410, "Gone", "UnrecoverableFileError",
5617 self.fileurls["dir-0share-json"]))
5618 def _check_unrecoverable_file(body):
5619 self.failIfIn("<html>", body)
5620 body = " ".join(body.strip().split())
5621 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5622 "could not be retrieved, because there were insufficient "
5623 "good shares. This might indicate that no servers were "
5624 "connected, insufficient servers were connected, the URI "
5625 "was corrupt, or that shares have been lost due to server "
5626 "departure, hard drive failure, or disk corruption. You "
5627 "should perform a filecheck on this object to learn more.")
5628 self.failUnlessReallyEqual(exp, body)
5629 d.addCallback(_check_unrecoverable_file)
5631 d.addCallback(lambda ignored:
5632 self.shouldHTTPError("GET dir-1share-json",
5633 410, "Gone", "UnrecoverableFileError",
5635 self.fileurls["dir-1share-json"]))
5636 d.addCallback(_check_unrecoverable_file)
5638 d.addCallback(lambda ignored:
5639 self.shouldHTTPError("GET imaginary",
5640 404, "Not Found", None,
5641 self.GET, self.fileurls["imaginary"]))
5643 # attach a webapi child that throws a random error, to test how it
5645 w = c0.getServiceNamed("webish")
5646 w.root.putChild("ERRORBOOM", ErrorBoom())
5648 # "Accept: */*" : should get a text/html stack trace
5649 # "Accept: text/plain" : should get a text/plain stack trace
5650 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5651 # no Accept header: should get a text/html stack trace
5653 d.addCallback(lambda ignored:
5654 self.shouldHTTPError("GET errorboom_html",
5655 500, "Internal Server Error", None,
5656 self.GET, "ERRORBOOM",
5657 headers={"accept": "*/*"}))
5658 def _internal_error_html1(body):
5659 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5660 d.addCallback(_internal_error_html1)
5662 d.addCallback(lambda ignored:
5663 self.shouldHTTPError("GET errorboom_text",
5664 500, "Internal Server Error", None,
5665 self.GET, "ERRORBOOM",
5666 headers={"accept": "text/plain"}))
5667 def _internal_error_text2(body):
5668 self.failIfIn("<html>", body)
5669 self.failUnless(body.startswith("Traceback "), body)
5670 d.addCallback(_internal_error_text2)
5672 CLI_accepts = "text/plain, application/octet-stream"
5673 d.addCallback(lambda ignored:
5674 self.shouldHTTPError("GET errorboom_text",
5675 500, "Internal Server Error", None,
5676 self.GET, "ERRORBOOM",
5677 headers={"accept": CLI_accepts}))
5678 def _internal_error_text3(body):
5679 self.failIfIn("<html>", body)
5680 self.failUnless(body.startswith("Traceback "), body)
5681 d.addCallback(_internal_error_text3)
5683 d.addCallback(lambda ignored:
5684 self.shouldHTTPError("GET errorboom_text",
5685 500, "Internal Server Error", None,
5686 self.GET, "ERRORBOOM"))
5687 def _internal_error_html4(body):
5688 self.failUnlessIn("<html>", body)
5689 d.addCallback(_internal_error_html4)
5691 def _flush_errors(res):
5692 # Trial: please ignore the CompletelyUnhandledError in the logs
5693 self.flushLoggedErrors(CompletelyUnhandledError)
5695 d.addBoth(_flush_errors)
5699 def test_blacklist(self):
5700 # download from a blacklisted URI, get an error
5701 self.basedir = "web/Grid/blacklist"
5703 c0 = self.g.clients[0]
5704 c0_basedir = c0.basedir
5705 fn = os.path.join(c0_basedir, "access.blacklist")
5707 DATA = "off-limits " * 50
5709 d = c0.upload(upload.Data(DATA, convergence=""))
5710 def _stash_uri_and_create_dir(ur):
5711 self.uri = ur.get_uri()
5712 self.url = "uri/"+self.uri
5713 u = uri.from_string_filenode(self.uri)
5714 self.si = u.get_storage_index()
5715 childnode = c0.create_node_from_uri(self.uri, None)
5716 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5717 d.addCallback(_stash_uri_and_create_dir)
5718 def _stash_dir(node):
5719 self.dir_node = node
5720 self.dir_uri = node.get_uri()
5721 self.dir_url = "uri/"+self.dir_uri
5722 d.addCallback(_stash_dir)
5723 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5724 def _check_dir_html(body):
5725 self.failUnlessIn("<html>", body)
5726 self.failUnlessIn("blacklisted.txt</a>", body)
5727 d.addCallback(_check_dir_html)
5728 d.addCallback(lambda ign: self.GET(self.url))
5729 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5731 def _blacklist(ign):
5733 f.write(" # this is a comment\n")
5735 f.write("\n") # also exercise blank lines
5736 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5738 # clients should be checking the blacklist each time, so we don't
5739 # need to restart the client
5740 d.addCallback(_blacklist)
5741 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5743 "Access Prohibited: off-limits",
5744 self.GET, self.url))
5746 # We should still be able to list the parent directory, in HTML...
5747 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5748 def _check_dir_html2(body):
5749 self.failUnlessIn("<html>", body)
5750 self.failUnlessIn("blacklisted.txt</strike>", body)
5751 d.addCallback(_check_dir_html2)
5753 # ... and in JSON (used by CLI).
5754 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5755 def _check_dir_json(res):
5756 data = simplejson.loads(res)
5757 self.failUnless(isinstance(data, list), data)
5758 self.failUnlessEqual(data[0], "dirnode")
5759 self.failUnless(isinstance(data[1], dict), data)
5760 self.failUnlessIn("children", data[1])
5761 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5762 childdata = data[1]["children"]["blacklisted.txt"]
5763 self.failUnless(isinstance(childdata, list), data)
5764 self.failUnlessEqual(childdata[0], "filenode")
5765 self.failUnless(isinstance(childdata[1], dict), data)
5766 d.addCallback(_check_dir_json)
5768 def _unblacklist(ign):
5769 open(fn, "w").close()
5770 # the Blacklist object watches mtime to tell when the file has
5771 # changed, but on windows this test will run faster than the
5772 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5773 # to force a reload.
5774 self.g.clients[0].blacklist.last_mtime -= 2.0
5775 d.addCallback(_unblacklist)
5777 # now a read should work
5778 d.addCallback(lambda ign: self.GET(self.url))
5779 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5781 # read again to exercise the blacklist-is-unchanged logic
5782 d.addCallback(lambda ign: self.GET(self.url))
5783 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5785 # now add a blacklisted directory, and make sure files under it are
5788 childnode = c0.create_node_from_uri(self.uri, None)
5789 return c0.create_dirnode({u"child": (childnode,{}) })
5790 d.addCallback(_add_dir)
5791 def _get_dircap(dn):
5792 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5793 self.dir_url_base = "uri/"+dn.get_write_uri()
5794 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5795 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5796 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5797 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5798 d.addCallback(_get_dircap)
5799 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5800 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5801 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5802 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5803 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5804 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5805 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5806 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5807 d.addCallback(lambda ign: self.GET(self.child_url))
5808 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5810 def _block_dir(ign):
5812 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5814 self.g.clients[0].blacklist.last_mtime -= 2.0
5815 d.addCallback(_block_dir)
5816 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5818 "Access Prohibited: dir-off-limits",
5819 self.GET, self.dir_url_base))
5820 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5822 "Access Prohibited: dir-off-limits",
5823 self.GET, self.dir_url_json1))
5824 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5826 "Access Prohibited: dir-off-limits",
5827 self.GET, self.dir_url_json2))
5828 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5830 "Access Prohibited: dir-off-limits",
5831 self.GET, self.dir_url_json_ro))
5832 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5834 "Access Prohibited: dir-off-limits",
5835 self.GET, self.child_url))
5839 class CompletelyUnhandledError(Exception):
5841 class ErrorBoom(rend.Page):
5842 def beforeRender(self, ctx):
5843 raise CompletelyUnhandledError("whoops")