1 import os.path, re, urllib, time, cgi
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow.util import escapeToXML
15 from nevow import rend
17 from allmydata import interfaces, uri, webish, dirnode
18 from allmydata.storage.shares import get_share_file
19 from allmydata.storage_client import StorageFarmBroker, StubServer
20 from allmydata.immutable import upload
21 from allmydata.immutable.downloader.status import DownloadStatus
22 from allmydata.dirnode import DirectoryNode
23 from allmydata.nodemaker import NodeMaker
24 from allmydata.unknown import UnknownNode
25 from allmydata.web import status, common
26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
27 from allmydata.util import fileutil, base32, hashutil
28 from allmydata.util.consumer import download_to_data
29 from allmydata.util.netstring import split_netstring
30 from allmydata.util.encodingutil import to_str
31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
32 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
33 make_mutable_file_uri, create_mutable_filenode
34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
35 from allmydata.mutable import servermap, publish, retrieve
36 import allmydata.test.common_util as testutil
37 from allmydata.test.no_network import GridTestMixin
38 from allmydata.test.common_web import HTTPClientGETFactory, \
40 from allmydata.client import Client, SecretHolder
41 from allmydata.introducer import IntroducerNode
43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
44 # create a webserver that works against them
46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
53 DIR_HTML_TAG = '<html lang="en">'
55 class FakeStatsProvider:
57 stats = {'stats': {}, 'counters': {}}
60 class FakeNodeMaker(NodeMaker):
65 'max_segment_size':128*1024 # 1024=KiB
67 def _create_lit(self, cap):
68 return FakeCHKFileNode(cap, self.all_contents)
69 def _create_immutable(self, cap):
70 return FakeCHKFileNode(cap, self.all_contents)
71 def _create_mutable(self, cap):
72 return FakeMutableFileNode(None, None,
73 self.encoding_params, None,
74 self.all_contents).init_from_cap(cap)
75 def create_mutable_file(self, contents="", keysize=None,
76 version=SDMF_VERSION):
77 n = FakeMutableFileNode(None, None, self.encoding_params, None,
79 return n.create(contents, version=version)
81 class FakeUploader(service.Service):
84 helper_connected = False
86 def upload(self, uploadable):
87 d = uploadable.get_size()
88 d.addCallback(lambda size: uploadable.read(size))
91 n = create_chk_filenode(data, self.all_contents)
92 ur = upload.UploadResults(file_size=len(data),
99 uri_extension_data={},
100 uri_extension_hash="fake",
101 verifycapstr="fakevcap")
102 ur.set_uri(n.get_uri())
104 d.addCallback(_got_data)
107 def get_helper_info(self):
108 return (self.helper_furl, self.helper_connected)
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeDisplayableServer(StubServer):
175 def __init__(self, serverid, nickname):
176 StubServer.__init__(self, serverid)
177 self.announcement = {"my-version": "allmydata-tahoe-fake",
178 "service-name": "storage",
179 "nickname": nickname}
180 def is_connected(self):
182 def get_permutation_seed(self):
184 def get_remote_host(self):
186 def get_last_loss_time(self):
188 def get_announcement_time(self):
190 def get_announcement(self):
191 return self.announcement
192 def get_nickname(self):
193 return self.announcement["nickname"]
194 def get_available_space(self):
197 class FakeBucketCounter(object):
199 return {"last-complete-bucket-count": 0}
200 def get_progress(self):
201 return {"estimated-time-per-cycle": 0,
202 "cycle-in-progress": False,
203 "remaining-wait-time": 0}
205 class FakeLeaseChecker(object):
207 self.expiration_enabled = False
209 self.override_lease_duration = None
210 self.sharetypes_to_expire = {}
212 return {"history": None}
213 def get_progress(self):
214 return {"estimated-time-per-cycle": 0,
215 "cycle-in-progress": False,
216 "remaining-wait-time": 0}
218 class FakeStorageServer(service.MultiService):
220 def __init__(self, nodeid, nickname):
221 service.MultiService.__init__(self)
222 self.my_nodeid = nodeid
223 self.nickname = nickname
224 self.bucket_counter = FakeBucketCounter()
225 self.lease_checker = FakeLeaseChecker()
227 return {"storage_server.accepting_immutable_shares": False}
229 class FakeClient(Client):
231 # don't upcall to Client.__init__, since we only want to initialize a
233 service.MultiService.__init__(self)
234 self.all_contents = {}
235 self.nodeid = "fake_nodeid"
236 self.nickname = u"fake_nickname \u263A"
237 self.introducer_furl = "None"
238 self.stats_provider = FakeStatsProvider()
239 self._secret_holder = SecretHolder("lease secret", "convergence secret")
241 self.convergence = "some random string"
242 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
243 # fake knowledge of another server
244 self.storage_broker.test_add_server("other_nodeid",
245 FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
246 self.introducer_client = None
247 self.history = FakeHistory()
248 self.uploader = FakeUploader()
249 self.uploader.all_contents = self.all_contents
250 self.uploader.setServiceParent(self)
251 self.blacklist = None
252 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
255 self.nodemaker.all_contents = self.all_contents
256 self.mutable_file_default = SDMF_VERSION
257 self.addService(FakeStorageServer(self.nodeid, self.nickname))
259 def get_long_nodeid(self):
261 def get_long_tubid(self):
264 def startService(self):
265 return service.MultiService.startService(self)
266 def stopService(self):
267 return service.MultiService.stopService(self)
269 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
271 class WebMixin(object):
273 self.s = FakeClient()
274 self.s.startService()
275 self.staticdir = self.mktemp()
277 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
279 self.ws.setServiceParent(self.s)
280 self.webish_port = self.ws.getPortnum()
281 self.webish_url = self.ws.getURL()
282 assert self.webish_url.endswith("/")
283 self.webish_url = self.webish_url[:-1] # these tests add their own /
285 l = [ self.s.create_dirnode() for x in range(6) ]
286 d = defer.DeferredList(l)
288 self.public_root = res[0][1]
289 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
290 self.public_url = "/uri/" + self.public_root.get_uri()
291 self.private_root = res[1][1]
295 self._foo_uri = foo.get_uri()
296 self._foo_readonly_uri = foo.get_readonly_uri()
297 self._foo_verifycap = foo.get_verify_cap().to_string()
298 # NOTE: we ignore the deferred on all set_uri() calls, because we
299 # know the fake nodes do these synchronously
300 self.public_root.set_uri(u"foo", foo.get_uri(),
301 foo.get_readonly_uri())
303 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
304 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
305 self._bar_txt_verifycap = n.get_verify_cap().to_string()
308 # XXX: Do we ever use this?
309 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
311 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
314 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
315 assert self._quux_txt_uri.startswith("URI:MDMF")
316 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
318 foo.set_uri(u"empty", res[3][1].get_uri(),
319 res[3][1].get_readonly_uri())
320 sub_uri = res[4][1].get_uri()
321 self._sub_uri = sub_uri
322 foo.set_uri(u"sub", sub_uri, sub_uri)
323 sub = self.s.create_node_from_uri(sub_uri)
326 _ign, n, blocking_uri = self.makefile(1)
327 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
329 # filenode to test for html encoding issues
330 self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
331 self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
332 self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
333 self._htmlname_escaped = escapeToXML(self._htmlname_raw)
334 self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
335 self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
336 self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
337 foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
339 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
340 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
341 # still think of it as an umlaut
342 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
344 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
345 self._baz_file_uri = baz_file
346 sub.set_uri(u"baz.txt", baz_file, baz_file)
348 _ign, n, self._bad_file_uri = self.makefile(3)
349 # this uri should not be downloadable
350 del self.s.all_contents[self._bad_file_uri]
353 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
354 rodir.get_readonly_uri())
355 rodir.set_uri(u"nor", baz_file, baz_file)
361 # public/foo/quux.txt
362 # public/foo/blockingfile
363 # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
366 # public/foo/sub/baz.txt
368 # public/reedownlee/nor
369 self.NEWFILE_CONTENTS = "newfile contents\n"
371 return foo.get_metadata_for(u"bar.txt")
373 def _got_metadata(metadata):
374 self._bar_txt_metadata = metadata
375 d.addCallback(_got_metadata)
378 def get_all_contents(self):
379 return self.s.all_contents
381 def makefile(self, number):
382 contents = "contents of file %s\n" % number
383 n = create_chk_filenode(contents, self.get_all_contents())
384 return contents, n, n.get_uri()
386 def makefile_mutable(self, number, mdmf=False):
387 contents = "contents of mutable file %s\n" % number
388 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
389 return contents, n, n.get_uri(), n.get_readonly_uri()
392 return self.s.stopService()
394 def failUnlessIsBarDotTxt(self, res):
395 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
397 def failUnlessIsQuuxDotTxt(self, res):
398 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
400 def failUnlessIsBazDotTxt(self, res):
401 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
403 def failUnlessIsSubBazDotTxt(self, res):
404 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
406 def failUnlessIsBarJSON(self, res):
407 data = simplejson.loads(res)
408 self.failUnless(isinstance(data, list))
409 self.failUnlessEqual(data[0], "filenode")
410 self.failUnless(isinstance(data[1], dict))
411 self.failIf(data[1]["mutable"])
412 self.failIfIn("rw_uri", data[1]) # immutable
413 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
414 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
415 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
417 def failUnlessIsQuuxJSON(self, res, readonly=False):
418 data = simplejson.loads(res)
419 self.failUnless(isinstance(data, list))
420 self.failUnlessEqual(data[0], "filenode")
421 self.failUnless(isinstance(data[1], dict))
423 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
425 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
426 self.failUnless(metadata['mutable'])
428 self.failIfIn("rw_uri", metadata)
430 self.failUnlessIn("rw_uri", metadata)
431 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
432 self.failUnlessIn("ro_uri", metadata)
433 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
434 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
436 def failUnlessIsFooJSON(self, res):
437 data = simplejson.loads(res)
438 self.failUnless(isinstance(data, list))
439 self.failUnlessEqual(data[0], "dirnode", res)
440 self.failUnless(isinstance(data[1], dict))
441 self.failUnless(data[1]["mutable"])
442 self.failUnlessIn("rw_uri", data[1]) # mutable
443 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
444 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
445 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
447 kidnames = sorted([unicode(n) for n in data[1]["children"]])
448 self.failUnlessEqual(kidnames,
449 [self._htmlname_unicode, u"bar.txt", u"baz.txt",
450 u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
451 kids = dict( [(unicode(name),value)
453 in data[1]["children"].iteritems()] )
454 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
455 self.failUnlessIn("metadata", kids[u"sub"][1])
456 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
457 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
458 self.failUnlessIn("linkcrtime", tahoe_md)
459 self.failUnlessIn("linkmotime", tahoe_md)
460 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
461 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
462 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
463 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
464 self._bar_txt_verifycap)
465 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
466 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
467 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
468 self._bar_txt_metadata["tahoe"]["linkcrtime"])
469 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
471 self.failUnlessIn("quux.txt", kids)
472 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
474 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
475 self._quux_txt_readonly_uri)
477 def GET(self, urlpath, followRedirect=False, return_response=False,
479 # if return_response=True, this fires with (data, statuscode,
480 # respheaders) instead of just data.
481 assert not isinstance(urlpath, unicode)
482 url = self.webish_url + urlpath
483 factory = HTTPClientGETFactory(url, method="GET",
484 followRedirect=followRedirect, **kwargs)
485 reactor.connectTCP("localhost", self.webish_port, factory)
488 return (data, factory.status, factory.response_headers)
490 d.addCallback(_got_data)
491 return factory.deferred
493 def HEAD(self, urlpath, return_response=False, **kwargs):
494 # this requires some surgery, because twisted.web.client doesn't want
495 # to give us back the response headers.
496 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
497 reactor.connectTCP("localhost", self.webish_port, factory)
500 return (data, factory.status, factory.response_headers)
502 d.addCallback(_got_data)
503 return factory.deferred
505 def PUT(self, urlpath, data, **kwargs):
506 url = self.webish_url + urlpath
507 return client.getPage(url, method="PUT", postdata=data, **kwargs)
509 def DELETE(self, urlpath):
510 url = self.webish_url + urlpath
511 return client.getPage(url, method="DELETE")
513 def POST(self, urlpath, followRedirect=False, **fields):
514 sepbase = "boogabooga"
518 form.append('Content-Disposition: form-data; name="_charset"')
522 for name, value in fields.iteritems():
523 if isinstance(value, tuple):
524 filename, value = value
525 form.append('Content-Disposition: form-data; name="%s"; '
526 'filename="%s"' % (name, filename.encode("utf-8")))
528 form.append('Content-Disposition: form-data; name="%s"' % name)
530 if isinstance(value, unicode):
531 value = value.encode("utf-8")
534 assert isinstance(value, str)
541 body = "\r\n".join(form) + "\r\n"
542 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
543 return self.POST2(urlpath, body, headers, followRedirect)
545 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
546 url = self.webish_url + urlpath
547 return client.getPage(url, method="POST", postdata=body,
548 headers=headers, followRedirect=followRedirect)
550 def shouldFail(self, res, expected_failure, which,
551 substring=None, response_substring=None):
552 if isinstance(res, failure.Failure):
553 res.trap(expected_failure)
555 self.failUnlessIn(substring, str(res), which)
556 if response_substring:
557 self.failUnlessIn(response_substring, res.value.response, which)
559 self.fail("%s was supposed to raise %s, not get '%s'" %
560 (which, expected_failure, res))
562 def shouldFail2(self, expected_failure, which, substring,
564 callable, *args, **kwargs):
565 assert substring is None or isinstance(substring, str)
566 assert response_substring is None or isinstance(response_substring, str)
567 d = defer.maybeDeferred(callable, *args, **kwargs)
569 if isinstance(res, failure.Failure):
570 res.trap(expected_failure)
572 self.failUnlessIn(substring, str(res),
573 "'%s' not in '%s' (response is '%s') for test '%s'" % \
574 (substring, str(res),
575 getattr(res.value, "response", ""),
577 if response_substring:
578 self.failUnlessIn(response_substring, res.value.response,
579 "'%s' not in '%s' for test '%s'" % \
580 (response_substring, res.value.response,
583 self.fail("%s was supposed to raise %s, not get '%s'" %
584 (which, expected_failure, res))
588 def should404(self, res, which):
589 if isinstance(res, failure.Failure):
590 res.trap(error.Error)
591 self.failUnlessReallyEqual(res.value.status, "404")
593 self.fail("%s was supposed to Error(404), not get '%s'" %
596 def should302(self, res, which):
597 if isinstance(res, failure.Failure):
598 res.trap(error.Error)
599 self.failUnlessReallyEqual(res.value.status, "302")
601 self.fail("%s was supposed to Error(302), not get '%s'" %
605 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
606 def test_create(self):
609 def test_welcome(self):
612 self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
613 self.failUnlessIn(FAVICON_MARKUP, res)
614 self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
615 self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
616 self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
617 self.failUnlessIn('Page rendered at', res)
618 self.failUnlessIn('Tahoe-LAFS code imported from:', res)
619 res_u = res.decode('utf-8')
620 self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
621 self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
622 self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
624 self.s.basedir = 'web/test_welcome'
625 fileutil.make_dirs("web/test_welcome")
626 fileutil.make_dirs("web/test_welcome/private")
628 d.addCallback(_check)
631 def test_introducer_status(self):
632 class MockIntroducerClient(object):
633 def __init__(self, connected):
634 self.connected = connected
635 def connected_to_introducer(self):
636 return self.connected
638 d = defer.succeed(None)
640 # introducer not connected, unguessable furl
641 def _set_introducer_not_connected_unguessable(ign):
642 self.s.introducer_furl = "pb://someIntroducer/secret"
643 self.s.introducer_client = MockIntroducerClient(False)
645 d.addCallback(_set_introducer_not_connected_unguessable)
646 def _check_introducer_not_connected_unguessable(res):
647 html = res.replace('\n', ' ')
648 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
649 self.failIfIn('pb://someIntroducer/secret', html)
650 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
651 d.addCallback(_check_introducer_not_connected_unguessable)
653 # introducer connected, unguessable furl
654 def _set_introducer_connected_unguessable(ign):
655 self.s.introducer_furl = "pb://someIntroducer/secret"
656 self.s.introducer_client = MockIntroducerClient(True)
658 d.addCallback(_set_introducer_connected_unguessable)
659 def _check_introducer_connected_unguessable(res):
660 html = res.replace('\n', ' ')
661 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
662 self.failIfIn('pb://someIntroducer/secret', html)
663 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
664 d.addCallback(_check_introducer_connected_unguessable)
666 # introducer connected, guessable furl
667 def _set_introducer_connected_guessable(ign):
668 self.s.introducer_furl = "pb://someIntroducer/introducer"
669 self.s.introducer_client = MockIntroducerClient(True)
671 d.addCallback(_set_introducer_connected_guessable)
672 def _check_introducer_connected_guessable(res):
673 html = res.replace('\n', ' ')
674 self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
675 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
676 d.addCallback(_check_introducer_connected_guessable)
679 def test_helper_status(self):
680 d = defer.succeed(None)
682 # set helper furl to None
683 def _set_no_helper(ign):
684 self.s.uploader.helper_furl = None
686 d.addCallback(_set_no_helper)
687 def _check_no_helper(res):
688 html = res.replace('\n', ' ')
689 self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
690 d.addCallback(_check_no_helper)
692 # enable helper, not connected
693 def _set_helper_not_connected(ign):
694 self.s.uploader.helper_furl = "pb://someHelper/secret"
695 self.s.uploader.helper_connected = False
697 d.addCallback(_set_helper_not_connected)
698 def _check_helper_not_connected(res):
699 html = res.replace('\n', ' ')
700 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
701 self.failIfIn('pb://someHelper/secret', html)
702 self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
703 d.addCallback(_check_helper_not_connected)
705 # enable helper, connected
706 def _set_helper_connected(ign):
707 self.s.uploader.helper_furl = "pb://someHelper/secret"
708 self.s.uploader.helper_connected = True
710 d.addCallback(_set_helper_connected)
711 def _check_helper_connected(res):
712 html = res.replace('\n', ' ')
713 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
714 self.failIfIn('pb://someHelper/secret', html)
715 self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
716 d.addCallback(_check_helper_connected)
719 def test_storage(self):
720 d = self.GET("/storage")
722 self.failUnlessIn('Storage Server Status', res)
723 self.failUnlessIn(FAVICON_MARKUP, res)
724 res_u = res.decode('utf-8')
725 self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
726 d.addCallback(_check)
729 def test_status(self):
730 h = self.s.get_history()
731 dl_num = h.list_all_download_statuses()[0].get_counter()
732 ul_num = h.list_all_upload_statuses()[0].get_counter()
733 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
734 pub_num = h.list_all_publish_statuses()[0].get_counter()
735 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
736 d = self.GET("/status", followRedirect=True)
738 self.failUnlessIn('Recent and Active Operations', res)
739 self.failUnlessIn('"down-%d"' % dl_num, res)
740 self.failUnlessIn('"up-%d"' % ul_num, res)
741 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
742 self.failUnlessIn('"publish-%d"' % pub_num, res)
743 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
744 d.addCallback(_check)
745 d.addCallback(lambda res: self.GET("/status/?t=json"))
746 def _check_json(res):
747 data = simplejson.loads(res)
748 self.failUnless(isinstance(data, dict))
749 #active = data["active"]
750 # TODO: test more. We need a way to fake an active operation
752 d.addCallback(_check_json)
754 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
756 self.failUnlessIn("File Download Status", res)
757 d.addCallback(_check_dl)
758 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
759 def _check_dl_json(res):
760 data = simplejson.loads(res)
761 self.failUnless(isinstance(data, dict))
762 self.failUnlessIn("read", data)
763 self.failUnlessEqual(data["read"][0]["length"], 120)
764 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
765 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
766 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
767 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
768 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
769 # serverids[] keys are strings, since that's what JSON does, but
770 # we'd really like them to be ints
771 self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
772 self.failUnless(data["serverids"].has_key("1"),
773 str(data["serverids"]))
774 self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
775 str(data["serverids"]))
776 self.failUnlessEqual(data["server_info"][phwr_id]["short"],
778 self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
780 self.failUnlessIn("dyhb", data)
781 self.failUnlessIn("misc", data)
782 d.addCallback(_check_dl_json)
783 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
785 self.failUnlessIn("File Upload Status", res)
786 d.addCallback(_check_ul)
787 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
788 def _check_mapupdate(res):
789 self.failUnlessIn("Mutable File Servermap Update Status", res)
790 d.addCallback(_check_mapupdate)
791 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
792 def _check_publish(res):
793 self.failUnlessIn("Mutable File Publish Status", res)
794 d.addCallback(_check_publish)
795 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
796 def _check_retrieve(res):
797 self.failUnlessIn("Mutable File Retrieve Status", res)
798 d.addCallback(_check_retrieve)
802 def test_status_numbers(self):
803 drrm = status.DownloadResultsRendererMixin()
804 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
805 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
806 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
807 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
808 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
809 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
810 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
811 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
812 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
814 urrm = status.UploadResultsRendererMixin()
815 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
816 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
817 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
818 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
819 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
820 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
821 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
822 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
823 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
825 def test_GET_FILEURL(self):
826 d = self.GET(self.public_url + "/foo/bar.txt")
827 d.addCallback(self.failUnlessIsBarDotTxt)
830 def test_GET_FILEURL_range(self):
831 headers = {"range": "bytes=1-10"}
832 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
833 return_response=True)
834 def _got((res, status, headers)):
835 self.failUnlessReallyEqual(int(status), 206)
836 self.failUnless(headers.has_key("content-range"))
837 self.failUnlessReallyEqual(headers["content-range"][0],
838 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
839 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
843 def test_GET_FILEURL_partial_range(self):
844 headers = {"range": "bytes=5-"}
845 length = len(self.BAR_CONTENTS)
846 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
847 return_response=True)
848 def _got((res, status, headers)):
849 self.failUnlessReallyEqual(int(status), 206)
850 self.failUnless(headers.has_key("content-range"))
851 self.failUnlessReallyEqual(headers["content-range"][0],
852 "bytes 5-%d/%d" % (length-1, length))
853 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
857 def test_GET_FILEURL_partial_end_range(self):
858 headers = {"range": "bytes=-5"}
859 length = len(self.BAR_CONTENTS)
860 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
861 return_response=True)
862 def _got((res, status, headers)):
863 self.failUnlessReallyEqual(int(status), 206)
864 self.failUnless(headers.has_key("content-range"))
865 self.failUnlessReallyEqual(headers["content-range"][0],
866 "bytes %d-%d/%d" % (length-5, length-1, length))
867 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
871 def test_GET_FILEURL_partial_range_overrun(self):
872 headers = {"range": "bytes=100-200"}
873 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
874 "416 Requested Range not satisfiable",
875 "First beyond end of file",
876 self.GET, self.public_url + "/foo/bar.txt",
880 def test_HEAD_FILEURL_range(self):
881 headers = {"range": "bytes=1-10"}
882 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
883 return_response=True)
884 def _got((res, status, headers)):
885 self.failUnlessReallyEqual(res, "")
886 self.failUnlessReallyEqual(int(status), 206)
887 self.failUnless(headers.has_key("content-range"))
888 self.failUnlessReallyEqual(headers["content-range"][0],
889 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
893 def test_HEAD_FILEURL_partial_range(self):
894 headers = {"range": "bytes=5-"}
895 length = len(self.BAR_CONTENTS)
896 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
897 return_response=True)
898 def _got((res, status, headers)):
899 self.failUnlessReallyEqual(int(status), 206)
900 self.failUnless(headers.has_key("content-range"))
901 self.failUnlessReallyEqual(headers["content-range"][0],
902 "bytes 5-%d/%d" % (length-1, length))
906 def test_HEAD_FILEURL_partial_end_range(self):
907 headers = {"range": "bytes=-5"}
908 length = len(self.BAR_CONTENTS)
909 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
910 return_response=True)
911 def _got((res, status, headers)):
912 self.failUnlessReallyEqual(int(status), 206)
913 self.failUnless(headers.has_key("content-range"))
914 self.failUnlessReallyEqual(headers["content-range"][0],
915 "bytes %d-%d/%d" % (length-5, length-1, length))
919 def test_HEAD_FILEURL_partial_range_overrun(self):
920 headers = {"range": "bytes=100-200"}
921 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
922 "416 Requested Range not satisfiable",
924 self.HEAD, self.public_url + "/foo/bar.txt",
928 def test_GET_FILEURL_range_bad(self):
929 headers = {"range": "BOGUS=fizbop-quarnak"}
930 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
931 return_response=True)
932 def _got((res, status, headers)):
933 self.failUnlessReallyEqual(int(status), 200)
934 self.failUnless(not headers.has_key("content-range"))
935 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
939 def test_HEAD_FILEURL(self):
940 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
941 def _got((res, status, headers)):
942 self.failUnlessReallyEqual(res, "")
943 self.failUnlessReallyEqual(headers["content-length"][0],
944 str(len(self.BAR_CONTENTS)))
945 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
949 def test_GET_FILEURL_named(self):
950 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
951 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
952 d = self.GET(base + "/@@name=/blah.txt")
953 d.addCallback(self.failUnlessIsBarDotTxt)
954 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
955 d.addCallback(self.failUnlessIsBarDotTxt)
956 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
957 d.addCallback(self.failUnlessIsBarDotTxt)
958 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
959 d.addCallback(self.failUnlessIsBarDotTxt)
960 save_url = base + "?save=true&filename=blah.txt"
961 d.addCallback(lambda res: self.GET(save_url))
962 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
963 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
964 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
965 u_url = base + "?save=true&filename=" + u_fn_e
966 d.addCallback(lambda res: self.GET(u_url))
967 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
970 def test_PUT_FILEURL_named_bad(self):
971 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
972 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
974 "/file can only be used with GET or HEAD",
975 self.PUT, base + "/@@name=/blah.txt", "")
979 def test_GET_DIRURL_named_bad(self):
980 base = "/file/%s" % urllib.quote(self._foo_uri)
981 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
984 self.GET, base + "/@@name=/blah.txt")
987 def test_GET_slash_file_bad(self):
988 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
990 "/file must be followed by a file-cap and a name",
994 def test_GET_unhandled_URI_named(self):
995 contents, n, newuri = self.makefile(12)
996 verifier_cap = n.get_verify_cap().to_string()
997 base = "/file/%s" % urllib.quote(verifier_cap)
998 # client.create_node_from_uri() can't handle verify-caps
999 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1000 "400 Bad Request", "is not a file-cap",
1004 def test_GET_unhandled_URI(self):
1005 contents, n, newuri = self.makefile(12)
1006 verifier_cap = n.get_verify_cap().to_string()
1007 base = "/uri/%s" % urllib.quote(verifier_cap)
1008 # client.create_node_from_uri() can't handle verify-caps
1009 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1011 "GET unknown URI type: can only do t=info",
1015 def test_GET_FILE_URI(self):
1016 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1018 d.addCallback(self.failUnlessIsBarDotTxt)
1021 def test_GET_FILE_URI_mdmf(self):
1022 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1024 d.addCallback(self.failUnlessIsQuuxDotTxt)
1027 def test_GET_FILE_URI_mdmf_extensions(self):
1028 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1030 d.addCallback(self.failUnlessIsQuuxDotTxt)
1033 def test_GET_FILE_URI_mdmf_readonly(self):
1034 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1036 d.addCallback(self.failUnlessIsQuuxDotTxt)
1039 def test_GET_FILE_URI_badchild(self):
1040 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1041 errmsg = "Files have no children, certainly not named 'boguschild'"
1042 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1043 "400 Bad Request", errmsg,
1047 def test_PUT_FILE_URI_badchild(self):
1048 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1049 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1050 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1051 "400 Bad Request", errmsg,
1055 def test_PUT_FILE_URI_mdmf(self):
1056 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1057 self._quux_new_contents = "new_contents"
1059 d.addCallback(lambda res:
1060 self.failUnlessIsQuuxDotTxt(res))
1061 d.addCallback(lambda ignored:
1062 self.PUT(base, self._quux_new_contents))
1063 d.addCallback(lambda ignored:
1065 d.addCallback(lambda res:
1066 self.failUnlessReallyEqual(res, self._quux_new_contents))
1069 def test_PUT_FILE_URI_mdmf_extensions(self):
1070 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1071 self._quux_new_contents = "new_contents"
1073 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1074 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1075 d.addCallback(lambda ignored: self.GET(base))
1076 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1080 def test_PUT_FILE_URI_mdmf_readonly(self):
1081 # We're not allowed to PUT things to a readonly cap.
1082 base = "/uri/%s" % self._quux_txt_readonly_uri
1084 d.addCallback(lambda res:
1085 self.failUnlessIsQuuxDotTxt(res))
1086 # What should we get here? We get a 500 error now; that's not right.
1087 d.addCallback(lambda ignored:
1088 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1089 "400 Bad Request", "read-only cap",
1090 self.PUT, base, "new data"))
1093 def test_PUT_FILE_URI_sdmf_readonly(self):
1094 # We're not allowed to put things to a readonly cap.
1095 base = "/uri/%s" % self._baz_txt_readonly_uri
1097 d.addCallback(lambda res:
1098 self.failUnlessIsBazDotTxt(res))
1099 d.addCallback(lambda ignored:
1100 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1101 "400 Bad Request", "read-only cap",
1102 self.PUT, base, "new_data"))
1105 def test_GET_etags(self):
1107 def _check_etags(uri):
1109 d2 = _get_etag(uri, 'json')
1110 d = defer.DeferredList([d1, d2], consumeErrors=True)
1111 def _check(results):
1112 # All deferred must succeed
1113 self.failUnless(all([r[0] for r in results]))
1114 # the etag for the t=json form should be just like the etag
1115 # fo the default t='' form, but with a 'json' suffix
1116 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1117 d.addCallback(_check)
1120 def _get_etag(uri, t=''):
1121 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1122 d = self.GET(targetbase, return_response=True, followRedirect=True)
1123 def _just_the_etag(result):
1124 data, response, headers = result
1125 etag = headers['etag'][0]
1126 if uri.startswith('URI:DIR'):
1127 self.failUnless(etag.startswith('DIR:'), etag)
1129 return d.addCallback(_just_the_etag)
1131 # Check that etags work with immutable directories
1132 (newkids, caps) = self._create_immutable_children()
1133 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1134 simplejson.dumps(newkids))
1135 def _stash_immdir_uri(uri):
1136 self._immdir_uri = uri
1138 d.addCallback(_stash_immdir_uri)
1139 d.addCallback(_check_etags)
1141 # Check that etags work with immutable files
1142 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1144 # use the ETag on GET
1145 def _check_match(ign):
1146 uri = "/uri/%s" % self._bar_txt_uri
1147 d = self.GET(uri, return_response=True)
1149 d.addCallback(lambda (data, code, headers):
1151 # do a GET that's supposed to match the ETag
1152 d.addCallback(lambda etag:
1153 self.GET(uri, return_response=True,
1154 headers={"If-None-Match": etag}))
1155 # make sure it short-circuited (304 instead of 200)
1156 d.addCallback(lambda (data, code, headers):
1157 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1159 d.addCallback(_check_match)
1161 def _no_etag(uri, t):
1162 target = "/uri/%s?t=%s" % (uri, t)
1163 d = self.GET(target, return_response=True, followRedirect=True)
1164 d.addCallback(lambda (data, code, headers):
1165 self.failIf("etag" in headers, target))
1167 def _yes_etag(uri, t):
1168 target = "/uri/%s?t=%s" % (uri, t)
1169 d = self.GET(target, return_response=True, followRedirect=True)
1170 d.addCallback(lambda (data, code, headers):
1171 self.failUnless("etag" in headers, target))
1174 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1175 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1176 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1177 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1178 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1180 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1181 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1182 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1183 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1184 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1185 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1189 # TODO: version of this with a Unicode filename
1190 def test_GET_FILEURL_save(self):
1191 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1192 return_response=True)
1193 def _got((res, statuscode, headers)):
1194 content_disposition = headers["content-disposition"][0]
1195 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1196 self.failUnlessIsBarDotTxt(res)
1200 def test_GET_FILEURL_missing(self):
1201 d = self.GET(self.public_url + "/foo/missing")
1202 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1205 def test_GET_FILEURL_info_mdmf(self):
1206 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1208 self.failUnlessIn("mutable file (mdmf)", res)
1209 self.failUnlessIn(self._quux_txt_uri, res)
1210 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1214 def test_GET_FILEURL_info_mdmf_readonly(self):
1215 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1217 self.failUnlessIn("mutable file (mdmf)", res)
1218 self.failIfIn(self._quux_txt_uri, res)
1219 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1223 def test_GET_FILEURL_info_sdmf(self):
1224 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1226 self.failUnlessIn("mutable file (sdmf)", res)
1227 self.failUnlessIn(self._baz_txt_uri, res)
1231 def test_GET_FILEURL_info_mdmf_extensions(self):
1232 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1234 self.failUnlessIn("mutable file (mdmf)", res)
1235 self.failUnlessIn(self._quux_txt_uri, res)
1236 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1240 def test_PUT_overwrite_only_files(self):
1241 # create a directory, put a file in that directory.
1242 contents, n, filecap = self.makefile(8)
1243 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1244 d.addCallback(lambda res:
1245 self.PUT(self.public_url + "/foo/dir/file1.txt",
1246 self.NEWFILE_CONTENTS))
1247 # try to overwrite the file with replace=only-files
1248 # (this should work)
1249 d.addCallback(lambda res:
1250 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1252 d.addCallback(lambda res:
1253 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1254 "There was already a child by that name, and you asked me "
1255 "to not replace it",
1256 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1260 def test_PUT_NEWFILEURL(self):
1261 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1262 # TODO: we lose the response code, so we can't check this
1263 #self.failUnlessReallyEqual(responsecode, 201)
1264 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1265 d.addCallback(lambda res:
1266 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1267 self.NEWFILE_CONTENTS))
1270 def test_PUT_NEWFILEURL_not_mutable(self):
1271 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1272 self.NEWFILE_CONTENTS)
1273 # TODO: we lose the response code, so we can't check this
1274 #self.failUnlessReallyEqual(responsecode, 201)
1275 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1276 d.addCallback(lambda res:
1277 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1278 self.NEWFILE_CONTENTS))
1281 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1282 # this should get us a few segments of an MDMF mutable file,
1283 # which we can then test for.
1284 contents = self.NEWFILE_CONTENTS * 300000
1285 d = self.PUT("/uri?format=mdmf",
1287 def _got_filecap(filecap):
1288 self.failUnless(filecap.startswith("URI:MDMF"))
1290 d.addCallback(_got_filecap)
1291 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1292 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1295 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1296 contents = self.NEWFILE_CONTENTS * 300000
1297 d = self.PUT("/uri?format=sdmf",
1299 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1300 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1303 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1304 contents = self.NEWFILE_CONTENTS * 300000
1305 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1306 400, "Bad Request", "Unknown format: foo",
1307 self.PUT, "/uri?format=foo",
1310 def test_PUT_NEWFILEURL_range_bad(self):
1311 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1312 target = self.public_url + "/foo/new.txt"
1313 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1314 "501 Not Implemented",
1315 "Content-Range in PUT not yet supported",
1316 # (and certainly not for immutable files)
1317 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1319 d.addCallback(lambda res:
1320 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1323 def test_PUT_NEWFILEURL_mutable(self):
1324 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1325 self.NEWFILE_CONTENTS)
1326 # TODO: we lose the response code, so we can't check this
1327 #self.failUnlessReallyEqual(responsecode, 201)
1328 def _check_uri(res):
1329 u = uri.from_string_mutable_filenode(res)
1330 self.failUnless(u.is_mutable())
1331 self.failIf(u.is_readonly())
1333 d.addCallback(_check_uri)
1334 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1335 d.addCallback(lambda res:
1336 self.failUnlessMutableChildContentsAre(self._foo_node,
1338 self.NEWFILE_CONTENTS))
1341 def test_PUT_NEWFILEURL_mutable_toobig(self):
1342 # It is okay to upload large mutable files, so we should be able
1344 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1345 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1348 def test_PUT_NEWFILEURL_replace(self):
1349 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1350 # TODO: we lose the response code, so we can't check this
1351 #self.failUnlessReallyEqual(responsecode, 200)
1352 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1353 d.addCallback(lambda res:
1354 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1355 self.NEWFILE_CONTENTS))
1358 def test_PUT_NEWFILEURL_bad_t(self):
1359 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1360 "PUT to a file: bad t=bogus",
1361 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1365 def test_PUT_NEWFILEURL_no_replace(self):
1366 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1367 self.NEWFILE_CONTENTS)
1368 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1370 "There was already a child by that name, and you asked me "
1371 "to not replace it")
1374 def test_PUT_NEWFILEURL_mkdirs(self):
1375 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1377 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1378 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1379 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1380 d.addCallback(lambda res:
1381 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1382 self.NEWFILE_CONTENTS))
1385 def test_PUT_NEWFILEURL_blocked(self):
1386 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1387 self.NEWFILE_CONTENTS)
1388 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1390 "Unable to create directory 'blockingfile': a file was in the way")
1393 def test_PUT_NEWFILEURL_emptyname(self):
1394 # an empty pathname component (i.e. a double-slash) is disallowed
1395 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1397 "The webapi does not allow empty pathname components",
1398 self.PUT, self.public_url + "/foo//new.txt", "")
1401 def test_DELETE_FILEURL(self):
1402 d = self.DELETE(self.public_url + "/foo/bar.txt")
1403 d.addCallback(lambda res:
1404 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1407 def test_DELETE_FILEURL_missing(self):
1408 d = self.DELETE(self.public_url + "/foo/missing")
1409 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1412 def test_DELETE_FILEURL_missing2(self):
1413 d = self.DELETE(self.public_url + "/missing/missing")
1414 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1417 def failUnlessHasBarDotTxtMetadata(self, res):
1418 data = simplejson.loads(res)
1419 self.failUnless(isinstance(data, list))
1420 self.failUnlessIn("metadata", data[1])
1421 self.failUnlessIn("tahoe", data[1]["metadata"])
1422 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1423 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1424 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1425 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1427 def test_GET_FILEURL_json(self):
1428 # twisted.web.http.parse_qs ignores any query args without an '=', so
1429 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1430 # instead. This may make it tricky to emulate the S3 interface
1432 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1434 self.failUnlessIsBarJSON(data)
1435 self.failUnlessHasBarDotTxtMetadata(data)
1437 d.addCallback(_check1)
1440 def test_GET_FILEURL_json_mutable_type(self):
1441 # The JSON should include format, which says whether the
1442 # file is SDMF or MDMF
1443 d = self.PUT("/uri?format=mdmf",
1444 self.NEWFILE_CONTENTS * 300000)
1445 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1446 def _got_json(json, version):
1447 data = simplejson.loads(json)
1448 assert "filenode" == data[0]
1450 assert isinstance(data, dict)
1452 self.failUnlessIn("format", data)
1453 self.failUnlessEqual(data["format"], version)
1455 d.addCallback(_got_json, "MDMF")
1456 # Now make an SDMF file and check that it is reported correctly.
1457 d.addCallback(lambda ignored:
1458 self.PUT("/uri?format=sdmf",
1459 self.NEWFILE_CONTENTS * 300000))
1460 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1461 d.addCallback(_got_json, "SDMF")
1464 def test_GET_FILEURL_json_mdmf(self):
1465 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1466 d.addCallback(self.failUnlessIsQuuxJSON)
1469 def test_GET_FILEURL_json_missing(self):
1470 d = self.GET(self.public_url + "/foo/missing?json")
1471 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1474 def test_GET_FILEURL_uri(self):
1475 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1477 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1478 d.addCallback(_check)
1479 d.addCallback(lambda res:
1480 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1482 # for now, for files, uris and readonly-uris are the same
1483 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1484 d.addCallback(_check2)
1487 def test_GET_FILEURL_badtype(self):
1488 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1491 self.public_url + "/foo/bar.txt?t=bogus")
1494 def test_CSS_FILE(self):
1495 d = self.GET("/tahoe.css", followRedirect=True)
1497 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1498 self.failUnless(CSS_STYLE.search(res), res)
1499 d.addCallback(_check)
1502 def test_GET_FILEURL_uri_missing(self):
1503 d = self.GET(self.public_url + "/foo/missing?t=uri")
1504 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1507 def _check_upload_and_mkdir_forms(self, html):
1508 # We should have a form to create a file, with radio buttons that allow
1509 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1510 self.failUnlessIn('name="t" value="upload"', html)
1511 self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1512 self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1513 self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1515 # We should also have the ability to create a mutable directory, with
1516 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1517 # or MDMF directory.
1518 self.failUnlessIn('name="t" value="mkdir"', html)
1519 self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1520 self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1522 self.failUnlessIn(FAVICON_MARKUP, html)
1524 def test_GET_DIRECTORY_html(self):
1525 d = self.GET(self.public_url + "/foo", followRedirect=True)
1527 self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1528 self._check_upload_and_mkdir_forms(html)
1529 self.failUnlessIn("quux", html)
1530 d.addCallback(_check)
1533 def test_GET_DIRECTORY_html_filenode_encoding(self):
1534 d = self.GET(self.public_url + "/foo", followRedirect=True)
1536 # Check if encoded entries are there
1537 self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1538 + self._htmlname_escaped + '</a>', html)
1539 self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1540 self.failIfIn(self._htmlname_escaped_double, html)
1541 # Make sure that Nevow escaping actually works by checking for unsafe characters
1542 # and that '&' is escaped.
1544 self.failUnlessIn(entity, self._htmlname_raw)
1545 self.failIfIn(entity, self._htmlname_escaped)
1546 self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1547 self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1548 d.addCallback(_check)
1551 def test_GET_root_html(self):
1553 d.addCallback(self._check_upload_and_mkdir_forms)
1556 def test_GET_DIRURL(self):
1557 # the addSlash means we get a redirect here
1558 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1560 d = self.GET(self.public_url + "/foo", followRedirect=True)
1562 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1564 # the FILE reference points to a URI, but it should end in bar.txt
1565 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1566 (ROOT, urllib.quote(self._bar_txt_uri)))
1567 get_bar = "".join([r'<td>FILE</td>',
1569 r'<a href="%s">bar.txt</a>' % bar_url,
1571 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1573 self.failUnless(re.search(get_bar, res), res)
1574 for label in ['unlink', 'rename/relink']:
1575 for line in res.split("\n"):
1576 # find the line that contains the relevant button for bar.txt
1577 if ("form action" in line and
1578 ('value="%s"' % (label,)) in line and
1579 'value="bar.txt"' in line):
1580 # the form target should use a relative URL
1581 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1582 self.failUnlessIn('action="%s"' % foo_url, line)
1583 # and the when_done= should too
1584 #done_url = urllib.quote(???)
1585 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1587 # 'unlink' needs to use POST because it directly has a side effect
1588 if label == 'unlink':
1589 self.failUnlessIn('method="post"', line)
1592 self.fail("unable to find '%s bar.txt' line" % (label,))
1594 # the DIR reference just points to a URI
1595 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1596 get_sub = ((r'<td>DIR</td>')
1597 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1598 self.failUnless(re.search(get_sub, res), res)
1599 d.addCallback(_check)
1601 # look at a readonly directory
1602 d.addCallback(lambda res:
1603 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1605 self.failUnlessIn("(read-only)", res)
1606 self.failIfIn("Upload a file", res)
1607 d.addCallback(_check2)
1609 # and at a directory that contains a readonly directory
1610 d.addCallback(lambda res:
1611 self.GET(self.public_url, followRedirect=True))
1613 self.failUnless(re.search('<td>DIR-RO</td>'
1614 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1615 d.addCallback(_check3)
1617 # and an empty directory
1618 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1620 self.failUnlessIn("directory is empty", res)
1621 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" class="btn" value="Create" />', re.I)
1622 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1623 d.addCallback(_check4)
1625 # and at a literal directory
1626 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1627 d.addCallback(lambda res:
1628 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1630 self.failUnlessIn('(immutable)', res)
1631 self.failUnless(re.search('<td>FILE</td>'
1632 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1633 d.addCallback(_check5)
1636 def test_GET_DIRURL_badtype(self):
1637 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1641 self.public_url + "/foo?t=bogus")
1644 def test_GET_DIRURL_json(self):
1645 d = self.GET(self.public_url + "/foo?t=json")
1646 d.addCallback(self.failUnlessIsFooJSON)
1649 def test_GET_DIRURL_json_format(self):
1650 d = self.PUT(self.public_url + \
1651 "/foo/sdmf.txt?format=sdmf",
1652 self.NEWFILE_CONTENTS * 300000)
1653 d.addCallback(lambda ignored:
1654 self.PUT(self.public_url + \
1655 "/foo/mdmf.txt?format=mdmf",
1656 self.NEWFILE_CONTENTS * 300000))
1657 # Now we have an MDMF and SDMF file in the directory. If we GET
1658 # its JSON, we should see their encodings.
1659 d.addCallback(lambda ignored:
1660 self.GET(self.public_url + "/foo?t=json"))
1661 def _got_json(json):
1662 data = simplejson.loads(json)
1663 assert data[0] == "dirnode"
1666 kids = data['children']
1668 mdmf_data = kids['mdmf.txt'][1]
1669 self.failUnlessIn("format", mdmf_data)
1670 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1672 sdmf_data = kids['sdmf.txt'][1]
1673 self.failUnlessIn("format", sdmf_data)
1674 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1675 d.addCallback(_got_json)
1679 def test_POST_DIRURL_manifest_no_ophandle(self):
1680 d = self.shouldFail2(error.Error,
1681 "test_POST_DIRURL_manifest_no_ophandle",
1683 "slow operation requires ophandle=",
1684 self.POST, self.public_url, t="start-manifest")
1687 def test_POST_DIRURL_manifest(self):
1688 d = defer.succeed(None)
1689 def getman(ignored, output):
1690 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1691 followRedirect=True)
1692 d.addCallback(self.wait_for_operation, "125")
1693 d.addCallback(self.get_operation_results, "125", output)
1695 d.addCallback(getman, None)
1696 def _got_html(manifest):
1697 self.failUnlessIn("Manifest of SI=", manifest)
1698 self.failUnlessIn("<td>sub</td>", manifest)
1699 self.failUnlessIn(self._sub_uri, manifest)
1700 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1701 self.failUnlessIn(FAVICON_MARKUP, manifest)
1702 d.addCallback(_got_html)
1704 # both t=status and unadorned GET should be identical
1705 d.addCallback(lambda res: self.GET("/operations/125"))
1706 d.addCallback(_got_html)
1708 d.addCallback(getman, "html")
1709 d.addCallback(_got_html)
1710 d.addCallback(getman, "text")
1711 def _got_text(manifest):
1712 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1713 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1714 d.addCallback(_got_text)
1715 d.addCallback(getman, "JSON")
1717 data = res["manifest"]
1719 for (path_list, cap) in data:
1720 got[tuple(path_list)] = cap
1721 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1722 self.failUnlessIn((u"sub", u"baz.txt"), got)
1723 self.failUnlessIn("finished", res)
1724 self.failUnlessIn("origin", res)
1725 self.failUnlessIn("storage-index", res)
1726 self.failUnlessIn("verifycaps", res)
1727 self.failUnlessIn("stats", res)
1728 d.addCallback(_got_json)
1731 def test_POST_DIRURL_deepsize_no_ophandle(self):
1732 d = self.shouldFail2(error.Error,
1733 "test_POST_DIRURL_deepsize_no_ophandle",
1735 "slow operation requires ophandle=",
1736 self.POST, self.public_url, t="start-deep-size")
1739 def test_POST_DIRURL_deepsize(self):
1740 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1741 followRedirect=True)
1742 d.addCallback(self.wait_for_operation, "126")
1743 d.addCallback(self.get_operation_results, "126", "json")
1744 def _got_json(data):
1745 self.failUnlessReallyEqual(data["finished"], True)
1747 self.failUnless(size > 1000)
1748 d.addCallback(_got_json)
1749 d.addCallback(self.get_operation_results, "126", "text")
1751 mo = re.search(r'^size: (\d+)$', res, re.M)
1752 self.failUnless(mo, res)
1753 size = int(mo.group(1))
1754 # with directories, the size varies.
1755 self.failUnless(size > 1000)
1756 d.addCallback(_got_text)
1759 def test_POST_DIRURL_deepstats_no_ophandle(self):
1760 d = self.shouldFail2(error.Error,
1761 "test_POST_DIRURL_deepstats_no_ophandle",
1763 "slow operation requires ophandle=",
1764 self.POST, self.public_url, t="start-deep-stats")
1767 def test_POST_DIRURL_deepstats(self):
1768 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1769 followRedirect=True)
1770 d.addCallback(self.wait_for_operation, "127")
1771 d.addCallback(self.get_operation_results, "127", "json")
1772 def _got_json(stats):
1773 expected = {"count-immutable-files": 4,
1774 "count-mutable-files": 2,
1775 "count-literal-files": 0,
1777 "count-directories": 3,
1778 "size-immutable-files": 76,
1779 "size-literal-files": 0,
1780 #"size-directories": 1912, # varies
1781 #"largest-directory": 1590,
1782 "largest-directory-children": 8,
1783 "largest-immutable-file": 19,
1785 for k,v in expected.iteritems():
1786 self.failUnlessReallyEqual(stats[k], v,
1787 "stats[%s] was %s, not %s" %
1789 self.failUnlessReallyEqual(stats["size-files-histogram"],
1791 d.addCallback(_got_json)
1794 def test_POST_DIRURL_stream_manifest(self):
1795 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1797 self.failUnless(res.endswith("\n"))
1798 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1799 self.failUnlessReallyEqual(len(units), 10)
1800 self.failUnlessEqual(units[-1]["type"], "stats")
1802 self.failUnlessEqual(first["path"], [])
1803 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1804 self.failUnlessEqual(first["type"], "directory")
1805 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1806 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1807 self.failIfEqual(baz["storage-index"], None)
1808 self.failIfEqual(baz["verifycap"], None)
1809 self.failIfEqual(baz["repaircap"], None)
1810 # XXX: Add quux and baz to this test.
1812 d.addCallback(_check)
1815 def test_GET_DIRURL_uri(self):
1816 d = self.GET(self.public_url + "/foo?t=uri")
1818 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1819 d.addCallback(_check)
1822 def test_GET_DIRURL_readonly_uri(self):
1823 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1825 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1826 d.addCallback(_check)
1829 def test_PUT_NEWDIRURL(self):
1830 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1831 d.addCallback(lambda res:
1832 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1833 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1834 d.addCallback(self.failUnlessNodeKeysAre, [])
1837 def test_PUT_NEWDIRURL_mdmf(self):
1838 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1839 d.addCallback(lambda res:
1840 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1841 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1842 d.addCallback(lambda node:
1843 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1846 def test_PUT_NEWDIRURL_sdmf(self):
1847 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1849 d.addCallback(lambda res:
1850 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1851 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1852 d.addCallback(lambda node:
1853 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1856 def test_PUT_NEWDIRURL_bad_format(self):
1857 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1858 400, "Bad Request", "Unknown format: foo",
1859 self.PUT, self.public_url +
1860 "/foo/newdir=?t=mkdir&format=foo", "")
1862 def test_POST_NEWDIRURL(self):
1863 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1864 d.addCallback(lambda res:
1865 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1866 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1867 d.addCallback(self.failUnlessNodeKeysAre, [])
1870 def test_POST_NEWDIRURL_mdmf(self):
1871 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1872 d.addCallback(lambda res:
1873 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1874 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875 d.addCallback(lambda node:
1876 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1879 def test_POST_NEWDIRURL_sdmf(self):
1880 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1881 d.addCallback(lambda res:
1882 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1883 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1884 d.addCallback(lambda node:
1885 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1888 def test_POST_NEWDIRURL_bad_format(self):
1889 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1890 400, "Bad Request", "Unknown format: foo",
1891 self.POST2, self.public_url + \
1892 "/foo/newdir?t=mkdir&format=foo", "")
1894 def test_POST_NEWDIRURL_emptyname(self):
1895 # an empty pathname component (i.e. a double-slash) is disallowed
1896 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1898 "The webapi does not allow empty pathname components, i.e. a double slash",
1899 self.POST, self.public_url + "//?t=mkdir")
1902 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1903 (newkids, caps) = self._create_initial_children()
1904 query = "/foo/newdir?t=mkdir-with-children"
1905 if version == MDMF_VERSION:
1906 query += "&format=mdmf"
1907 elif version == SDMF_VERSION:
1908 query += "&format=sdmf"
1910 version = SDMF_VERSION # for later
1911 d = self.POST2(self.public_url + query,
1912 simplejson.dumps(newkids))
1914 n = self.s.create_node_from_uri(uri.strip())
1915 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1916 self.failUnlessEqual(n._node.get_version(), version)
1917 d2.addCallback(lambda ign:
1918 self.failUnlessROChildURIIs(n, u"child-imm",
1920 d2.addCallback(lambda ign:
1921 self.failUnlessRWChildURIIs(n, u"child-mutable",
1923 d2.addCallback(lambda ign:
1924 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1926 d2.addCallback(lambda ign:
1927 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1928 caps['unknown_rocap']))
1929 d2.addCallback(lambda ign:
1930 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1931 caps['unknown_rwcap']))
1932 d2.addCallback(lambda ign:
1933 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1934 caps['unknown_immcap']))
1935 d2.addCallback(lambda ign:
1936 self.failUnlessRWChildURIIs(n, u"dirchild",
1938 d2.addCallback(lambda ign:
1939 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1941 d2.addCallback(lambda ign:
1942 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1943 caps['emptydircap']))
1945 d.addCallback(_check)
1946 d.addCallback(lambda res:
1947 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1948 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1949 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1950 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1951 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1954 def test_POST_NEWDIRURL_initial_children(self):
1955 return self._do_POST_NEWDIRURL_initial_children_test()
1957 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1958 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1960 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1961 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1963 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1964 (newkids, caps) = self._create_initial_children()
1965 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1966 400, "Bad Request", "Unknown format: foo",
1967 self.POST2, self.public_url + \
1968 "/foo/newdir?t=mkdir-with-children&format=foo",
1969 simplejson.dumps(newkids))
1971 def test_POST_NEWDIRURL_immutable(self):
1972 (newkids, caps) = self._create_immutable_children()
1973 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1974 simplejson.dumps(newkids))
1976 n = self.s.create_node_from_uri(uri.strip())
1977 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1978 d2.addCallback(lambda ign:
1979 self.failUnlessROChildURIIs(n, u"child-imm",
1981 d2.addCallback(lambda ign:
1982 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1983 caps['unknown_immcap']))
1984 d2.addCallback(lambda ign:
1985 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1987 d2.addCallback(lambda ign:
1988 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1990 d2.addCallback(lambda ign:
1991 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1992 caps['emptydircap']))
1994 d.addCallback(_check)
1995 d.addCallback(lambda res:
1996 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1997 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1998 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2005 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2006 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2007 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2008 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2009 d.addErrback(self.explain_web_error)
2012 def test_POST_NEWDIRURL_immutable_bad(self):
2013 (newkids, caps) = self._create_initial_children()
2014 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2016 "needed to be immutable but was not",
2018 self.public_url + "/foo/newdir?t=mkdir-immutable",
2019 simplejson.dumps(newkids))
2022 def test_PUT_NEWDIRURL_exists(self):
2023 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2024 d.addCallback(lambda res:
2025 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2026 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2027 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2030 def test_PUT_NEWDIRURL_blocked(self):
2031 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2032 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2034 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2035 d.addCallback(lambda res:
2036 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2037 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2038 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2041 def test_PUT_NEWDIRURL_mkdirs(self):
2042 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2043 d.addCallback(lambda res:
2044 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2045 d.addCallback(lambda res:
2046 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2047 d.addCallback(lambda res:
2048 self._foo_node.get_child_at_path(u"subdir/newdir"))
2049 d.addCallback(self.failUnlessNodeKeysAre, [])
2052 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2053 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2054 d.addCallback(lambda ignored:
2055 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2056 d.addCallback(lambda ignored:
2057 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2058 d.addCallback(lambda ignored:
2059 self._foo_node.get_child_at_path(u"subdir"))
2060 def _got_subdir(subdir):
2061 # XXX: What we want?
2062 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2063 self.failUnlessNodeHasChild(subdir, u"newdir")
2064 return subdir.get_child_at_path(u"newdir")
2065 d.addCallback(_got_subdir)
2066 d.addCallback(lambda newdir:
2067 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2070 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2071 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2072 d.addCallback(lambda ignored:
2073 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2074 d.addCallback(lambda ignored:
2075 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2076 d.addCallback(lambda ignored:
2077 self._foo_node.get_child_at_path(u"subdir"))
2078 def _got_subdir(subdir):
2079 # XXX: What we want?
2080 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2081 self.failUnlessNodeHasChild(subdir, u"newdir")
2082 return subdir.get_child_at_path(u"newdir")
2083 d.addCallback(_got_subdir)
2084 d.addCallback(lambda newdir:
2085 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2088 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2089 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2090 400, "Bad Request", "Unknown format: foo",
2091 self.PUT, self.public_url + \
2092 "/foo/subdir/newdir?t=mkdir&format=foo",
2095 def test_DELETE_DIRURL(self):
2096 d = self.DELETE(self.public_url + "/foo")
2097 d.addCallback(lambda res:
2098 self.failIfNodeHasChild(self.public_root, u"foo"))
2101 def test_DELETE_DIRURL_missing(self):
2102 d = self.DELETE(self.public_url + "/foo/missing")
2103 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2104 d.addCallback(lambda res:
2105 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2108 def test_DELETE_DIRURL_missing2(self):
2109 d = self.DELETE(self.public_url + "/missing")
2110 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2113 def dump_root(self):
2115 w = webish.DirnodeWalkerMixin()
2116 def visitor(childpath, childnode, metadata):
2118 d = w.walk(self.public_root, visitor)
2121 def failUnlessNodeKeysAre(self, node, expected_keys):
2122 for k in expected_keys:
2123 assert isinstance(k, unicode)
2125 def _check(children):
2126 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2127 d.addCallback(_check)
2129 def failUnlessNodeHasChild(self, node, name):
2130 assert isinstance(name, unicode)
2132 def _check(children):
2133 self.failUnlessIn(name, children)
2134 d.addCallback(_check)
2136 def failIfNodeHasChild(self, node, name):
2137 assert isinstance(name, unicode)
2139 def _check(children):
2140 self.failIfIn(name, children)
2141 d.addCallback(_check)
2144 def failUnlessChildContentsAre(self, node, name, expected_contents):
2145 assert isinstance(name, unicode)
2146 d = node.get_child_at_path(name)
2147 d.addCallback(lambda node: download_to_data(node))
2148 def _check(contents):
2149 self.failUnlessReallyEqual(contents, expected_contents)
2150 d.addCallback(_check)
2153 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2154 assert isinstance(name, unicode)
2155 d = node.get_child_at_path(name)
2156 d.addCallback(lambda node: node.download_best_version())
2157 def _check(contents):
2158 self.failUnlessReallyEqual(contents, expected_contents)
2159 d.addCallback(_check)
2162 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2163 assert isinstance(name, unicode)
2164 d = node.get_child_at_path(name)
2166 self.failUnless(child.is_unknown() or not child.is_readonly())
2167 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2168 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2169 expected_ro_uri = self._make_readonly(expected_uri)
2171 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2172 d.addCallback(_check)
2175 def failUnlessROChildURIIs(self, node, name, expected_uri):
2176 assert isinstance(name, unicode)
2177 d = node.get_child_at_path(name)
2179 self.failUnless(child.is_unknown() or child.is_readonly())
2180 self.failUnlessReallyEqual(child.get_write_uri(), None)
2181 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2182 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2183 d.addCallback(_check)
2186 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2187 assert isinstance(name, unicode)
2188 d = node.get_child_at_path(name)
2190 self.failUnless(child.is_unknown() or not child.is_readonly())
2191 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2192 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2193 expected_ro_uri = self._make_readonly(got_uri)
2195 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2196 d.addCallback(_check)
2199 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2200 assert isinstance(name, unicode)
2201 d = node.get_child_at_path(name)
2203 self.failUnless(child.is_unknown() or child.is_readonly())
2204 self.failUnlessReallyEqual(child.get_write_uri(), None)
2205 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2206 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2207 d.addCallback(_check)
2210 def failUnlessCHKURIHasContents(self, got_uri, contents):
2211 self.failUnless(self.get_all_contents()[got_uri] == contents)
2213 def test_POST_upload(self):
2214 d = self.POST(self.public_url + "/foo", t="upload",
2215 file=("new.txt", self.NEWFILE_CONTENTS))
2217 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2218 d.addCallback(lambda res:
2219 self.failUnlessChildContentsAre(fn, u"new.txt",
2220 self.NEWFILE_CONTENTS))
2223 def test_POST_upload_unicode(self):
2224 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2225 d = self.POST(self.public_url + "/foo", t="upload",
2226 file=(filename, self.NEWFILE_CONTENTS))
2228 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2229 d.addCallback(lambda res:
2230 self.failUnlessChildContentsAre(fn, filename,
2231 self.NEWFILE_CONTENTS))
2232 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2233 d.addCallback(lambda res: self.GET(target_url))
2234 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2235 self.NEWFILE_CONTENTS,
2239 def test_POST_upload_unicode_named(self):
2240 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2241 d = self.POST(self.public_url + "/foo", t="upload",
2243 file=("overridden", self.NEWFILE_CONTENTS))
2245 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2246 d.addCallback(lambda res:
2247 self.failUnlessChildContentsAre(fn, filename,
2248 self.NEWFILE_CONTENTS))
2249 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2250 d.addCallback(lambda res: self.GET(target_url))
2251 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2252 self.NEWFILE_CONTENTS,
2256 def test_POST_upload_no_link(self):
2257 d = self.POST("/uri", t="upload",
2258 file=("new.txt", self.NEWFILE_CONTENTS))
2259 def _check_upload_results(page):
2260 # this should be a page which describes the results of the upload
2261 # that just finished.
2262 self.failUnlessIn("Upload Results:", page)
2263 self.failUnlessIn("URI:", page)
2264 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2265 mo = uri_re.search(page)
2266 self.failUnless(mo, page)
2267 new_uri = mo.group(1)
2269 d.addCallback(_check_upload_results)
2270 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2273 def test_POST_upload_no_link_whendone(self):
2274 d = self.POST("/uri", t="upload", when_done="/",
2275 file=("new.txt", self.NEWFILE_CONTENTS))
2276 d.addBoth(self.shouldRedirect, "/")
2279 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2280 d = defer.maybeDeferred(callable, *args, **kwargs)
2282 if isinstance(res, failure.Failure):
2283 res.trap(error.PageRedirect)
2284 statuscode = res.value.status
2285 target = res.value.location
2286 return checker(statuscode, target)
2287 self.fail("%s: callable was supposed to redirect, not return '%s'"
2292 def test_POST_upload_no_link_whendone_results(self):
2293 def check(statuscode, target):
2294 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2295 self.failUnless(target.startswith(self.webish_url), target)
2296 return client.getPage(target, method="GET")
2297 # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2298 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2300 self.POST, "/uri", t="upload",
2301 when_done="/%75ri/%(uri)s",
2302 file=("new.txt", self.NEWFILE_CONTENTS))
2303 d.addCallback(lambda res:
2304 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2307 def test_POST_upload_no_link_mutable(self):
2308 d = self.POST("/uri", t="upload", mutable="true",
2309 file=("new.txt", self.NEWFILE_CONTENTS))
2310 def _check(filecap):
2311 filecap = filecap.strip()
2312 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2313 self.filecap = filecap
2314 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2315 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2316 n = self.s.create_node_from_uri(filecap)
2317 return n.download_best_version()
2318 d.addCallback(_check)
2320 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2321 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2322 d.addCallback(_check2)
2324 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2325 return self.GET("/file/%s" % urllib.quote(self.filecap))
2326 d.addCallback(_check3)
2328 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2329 d.addCallback(_check4)
2332 def test_POST_upload_no_link_mutable_toobig(self):
2333 # The SDMF size limit is no longer in place, so we should be
2334 # able to upload mutable files that are as large as we want them
2336 d = self.POST("/uri", t="upload", mutable="true",
2337 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2341 def test_POST_upload_format_unlinked(self):
2342 def _check_upload_unlinked(ign, format, uri_prefix):
2343 filename = format + ".txt"
2344 d = self.POST("/uri?t=upload&format=" + format,
2345 file=(filename, self.NEWFILE_CONTENTS * 300000))
2346 def _got_results(results):
2347 if format.upper() in ("SDMF", "MDMF"):
2348 # webapi.rst says this returns a filecap
2351 # for immutable, it returns an "upload results page", and
2352 # the filecap is buried inside
2353 line = [l for l in results.split("\n") if "URI: " in l][0]
2354 mo = re.search(r'<span>([^<]+)</span>', line)
2355 filecap = mo.group(1)
2356 self.failUnless(filecap.startswith(uri_prefix),
2357 (uri_prefix, filecap))
2358 return self.GET("/uri/%s?t=json" % filecap)
2359 d.addCallback(_got_results)
2360 def _got_json(json):
2361 data = simplejson.loads(json)
2363 self.failUnlessIn("format", data)
2364 self.failUnlessEqual(data["format"], format.upper())
2365 d.addCallback(_got_json)
2367 d = defer.succeed(None)
2368 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2369 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2370 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2371 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2374 def test_POST_upload_bad_format_unlinked(self):
2375 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2376 400, "Bad Request", "Unknown format: foo",
2378 "/uri?t=upload&format=foo",
2379 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2381 def test_POST_upload_format(self):
2382 def _check_upload(ign, format, uri_prefix, fn=None):
2383 filename = format + ".txt"
2384 d = self.POST(self.public_url +
2385 "/foo?t=upload&format=" + format,
2386 file=(filename, self.NEWFILE_CONTENTS * 300000))
2387 def _got_filecap(filecap):
2389 filenameu = unicode(filename)
2390 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2391 self.failUnless(filecap.startswith(uri_prefix))
2392 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2393 d.addCallback(_got_filecap)
2394 def _got_json(json):
2395 data = simplejson.loads(json)
2397 self.failUnlessIn("format", data)
2398 self.failUnlessEqual(data["format"], format.upper())
2399 d.addCallback(_got_json)
2402 d = defer.succeed(None)
2403 d.addCallback(_check_upload, "chk", "URI:CHK")
2404 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2405 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2406 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2409 def test_POST_upload_bad_format(self):
2410 return self.shouldHTTPError("POST_upload_bad_format",
2411 400, "Bad Request", "Unknown format: foo",
2412 self.POST, self.public_url + \
2413 "/foo?t=upload&format=foo",
2414 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2416 def test_POST_upload_mutable(self):
2417 # this creates a mutable file
2418 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2419 file=("new.txt", self.NEWFILE_CONTENTS))
2421 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2422 d.addCallback(lambda res:
2423 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2424 self.NEWFILE_CONTENTS))
2425 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2427 self.failUnless(IMutableFileNode.providedBy(newnode))
2428 self.failUnless(newnode.is_mutable())
2429 self.failIf(newnode.is_readonly())
2430 self._mutable_node = newnode
2431 self._mutable_uri = newnode.get_uri()
2434 # now upload it again and make sure that the URI doesn't change
2435 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2436 d.addCallback(lambda res:
2437 self.POST(self.public_url + "/foo", t="upload",
2439 file=("new.txt", NEWER_CONTENTS)))
2440 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2441 d.addCallback(lambda res:
2442 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2444 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2446 self.failUnless(IMutableFileNode.providedBy(newnode))
2447 self.failUnless(newnode.is_mutable())
2448 self.failIf(newnode.is_readonly())
2449 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2450 d.addCallback(_got2)
2452 # upload a second time, using PUT instead of POST
2453 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2454 d.addCallback(lambda res:
2455 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2456 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2457 d.addCallback(lambda res:
2458 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2461 # finally list the directory, since mutable files are displayed
2462 # slightly differently
2464 d.addCallback(lambda res:
2465 self.GET(self.public_url + "/foo/",
2466 followRedirect=True))
2467 def _check_page(res):
2468 # TODO: assert more about the contents
2469 self.failUnlessIn("SSK", res)
2471 d.addCallback(_check_page)
2473 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2475 self.failUnless(IMutableFileNode.providedBy(newnode))
2476 self.failUnless(newnode.is_mutable())
2477 self.failIf(newnode.is_readonly())
2478 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2479 d.addCallback(_got3)
2481 # look at the JSON form of the enclosing directory
2482 d.addCallback(lambda res:
2483 self.GET(self.public_url + "/foo/?t=json",
2484 followRedirect=True))
2485 def _check_page_json(res):
2486 parsed = simplejson.loads(res)
2487 self.failUnlessEqual(parsed[0], "dirnode")
2488 children = dict( [(unicode(name),value)
2490 in parsed[1]["children"].iteritems()] )
2491 self.failUnlessIn(u"new.txt", children)
2492 new_json = children[u"new.txt"]
2493 self.failUnlessEqual(new_json[0], "filenode")
2494 self.failUnless(new_json[1]["mutable"])
2495 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2496 ro_uri = self._mutable_node.get_readonly().to_string()
2497 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2498 d.addCallback(_check_page_json)
2500 # and the JSON form of the file
2501 d.addCallback(lambda res:
2502 self.GET(self.public_url + "/foo/new.txt?t=json"))
2503 def _check_file_json(res):
2504 parsed = simplejson.loads(res)
2505 self.failUnlessEqual(parsed[0], "filenode")
2506 self.failUnless(parsed[1]["mutable"])
2507 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2508 ro_uri = self._mutable_node.get_readonly().to_string()
2509 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2510 d.addCallback(_check_file_json)
2512 # and look at t=uri and t=readonly-uri
2513 d.addCallback(lambda res:
2514 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2515 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2516 d.addCallback(lambda res:
2517 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2518 def _check_ro_uri(res):
2519 ro_uri = self._mutable_node.get_readonly().to_string()
2520 self.failUnlessReallyEqual(res, ro_uri)
2521 d.addCallback(_check_ro_uri)
2523 # make sure we can get to it from /uri/URI
2524 d.addCallback(lambda res:
2525 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2526 d.addCallback(lambda res:
2527 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2529 # and that HEAD computes the size correctly
2530 d.addCallback(lambda res:
2531 self.HEAD(self.public_url + "/foo/new.txt",
2532 return_response=True))
2533 def _got_headers((res, status, headers)):
2534 self.failUnlessReallyEqual(res, "")
2535 self.failUnlessReallyEqual(headers["content-length"][0],
2536 str(len(NEW2_CONTENTS)))
2537 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2538 d.addCallback(_got_headers)
2540 # make sure that outdated size limits aren't enforced anymore.
2541 d.addCallback(lambda ignored:
2542 self.POST(self.public_url + "/foo", t="upload",
2545 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2546 d.addErrback(self.dump_error)
2549 def test_POST_upload_mutable_toobig(self):
2550 # SDMF had a size limti that was removed a while ago. MDMF has
2551 # never had a size limit. Test to make sure that we do not
2552 # encounter errors when trying to upload large mutable files,
2553 # since there should be no coded prohibitions regarding large
2555 d = self.POST(self.public_url + "/foo",
2556 t="upload", mutable="true",
2557 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2560 def dump_error(self, f):
2561 # if the web server returns an error code (like 400 Bad Request),
2562 # web.client.getPage puts the HTTP response body into the .response
2563 # attribute of the exception object that it gives back. It does not
2564 # appear in the Failure's repr(), so the ERROR that trial displays
2565 # will be rather terse and unhelpful. addErrback this method to the
2566 # end of your chain to get more information out of these errors.
2567 if f.check(error.Error):
2568 print "web.error.Error:"
2570 print f.value.response
2573 def test_POST_upload_replace(self):
2574 d = self.POST(self.public_url + "/foo", t="upload",
2575 file=("bar.txt", self.NEWFILE_CONTENTS))
2577 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2578 d.addCallback(lambda res:
2579 self.failUnlessChildContentsAre(fn, u"bar.txt",
2580 self.NEWFILE_CONTENTS))
2583 def test_POST_upload_no_replace_ok(self):
2584 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2585 file=("new.txt", self.NEWFILE_CONTENTS))
2586 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2587 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2588 self.NEWFILE_CONTENTS))
2591 def test_POST_upload_no_replace_queryarg(self):
2592 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2593 file=("bar.txt", self.NEWFILE_CONTENTS))
2594 d.addBoth(self.shouldFail, error.Error,
2595 "POST_upload_no_replace_queryarg",
2597 "There was already a child by that name, and you asked me "
2598 "to not replace it")
2599 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2600 d.addCallback(self.failUnlessIsBarDotTxt)
2603 def test_POST_upload_no_replace_field(self):
2604 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2605 file=("bar.txt", self.NEWFILE_CONTENTS))
2606 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2608 "There was already a child by that name, and you asked me "
2609 "to not replace it")
2610 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2611 d.addCallback(self.failUnlessIsBarDotTxt)
2614 def test_POST_upload_whendone(self):
2615 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2616 file=("new.txt", self.NEWFILE_CONTENTS))
2617 d.addBoth(self.shouldRedirect, "/THERE")
2619 d.addCallback(lambda res:
2620 self.failUnlessChildContentsAre(fn, u"new.txt",
2621 self.NEWFILE_CONTENTS))
2624 def test_POST_upload_named(self):
2626 d = self.POST(self.public_url + "/foo", t="upload",
2627 name="new.txt", file=self.NEWFILE_CONTENTS)
2628 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2629 d.addCallback(lambda res:
2630 self.failUnlessChildContentsAre(fn, u"new.txt",
2631 self.NEWFILE_CONTENTS))
2634 def test_POST_upload_named_badfilename(self):
2635 d = self.POST(self.public_url + "/foo", t="upload",
2636 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2637 d.addBoth(self.shouldFail, error.Error,
2638 "test_POST_upload_named_badfilename",
2640 "name= may not contain a slash",
2642 # make sure that nothing was added
2643 d.addCallback(lambda res:
2644 self.failUnlessNodeKeysAre(self._foo_node,
2645 [self._htmlname_unicode,
2646 u"bar.txt", u"baz.txt", u"blockingfile",
2647 u"empty", u"n\u00fc.txt", u"quux.txt",
2651 def test_POST_FILEURL_check(self):
2652 bar_url = self.public_url + "/foo/bar.txt"
2653 d = self.POST(bar_url, t="check")
2655 self.failUnlessIn("Healthy :", res)
2656 d.addCallback(_check)
2657 redir_url = "http://allmydata.org/TARGET"
2658 def _check2(statuscode, target):
2659 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2660 self.failUnlessReallyEqual(target, redir_url)
2661 d.addCallback(lambda res:
2662 self.shouldRedirect2("test_POST_FILEURL_check",
2666 when_done=redir_url))
2667 d.addCallback(lambda res:
2668 self.POST(bar_url, t="check", return_to=redir_url))
2670 self.failUnlessIn("Healthy :", res)
2671 self.failUnlessIn("Return to file", res)
2672 self.failUnlessIn(redir_url, res)
2673 d.addCallback(_check3)
2675 d.addCallback(lambda res:
2676 self.POST(bar_url, t="check", output="JSON"))
2677 def _check_json(res):
2678 data = simplejson.loads(res)
2679 self.failUnlessIn("storage-index", data)
2680 self.failUnless(data["results"]["healthy"])
2681 d.addCallback(_check_json)
2685 def test_POST_FILEURL_check_and_repair(self):
2686 bar_url = self.public_url + "/foo/bar.txt"
2687 d = self.POST(bar_url, t="check", repair="true")
2689 self.failUnlessIn("Healthy :", res)
2690 d.addCallback(_check)
2691 redir_url = "http://allmydata.org/TARGET"
2692 def _check2(statuscode, target):
2693 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2694 self.failUnlessReallyEqual(target, redir_url)
2695 d.addCallback(lambda res:
2696 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2699 t="check", repair="true",
2700 when_done=redir_url))
2701 d.addCallback(lambda res:
2702 self.POST(bar_url, t="check", return_to=redir_url))
2704 self.failUnlessIn("Healthy :", res)
2705 self.failUnlessIn("Return to file", res)
2706 self.failUnlessIn(redir_url, res)
2707 d.addCallback(_check3)
2710 def test_POST_DIRURL_check(self):
2711 foo_url = self.public_url + "/foo/"
2712 d = self.POST(foo_url, t="check")
2714 self.failUnlessIn("Healthy :", res)
2715 d.addCallback(_check)
2716 redir_url = "http://allmydata.org/TARGET"
2717 def _check2(statuscode, target):
2718 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2719 self.failUnlessReallyEqual(target, redir_url)
2720 d.addCallback(lambda res:
2721 self.shouldRedirect2("test_POST_DIRURL_check",
2725 when_done=redir_url))
2726 d.addCallback(lambda res:
2727 self.POST(foo_url, t="check", return_to=redir_url))
2729 self.failUnlessIn("Healthy :", res)
2730 self.failUnlessIn("Return to file/directory", res)
2731 self.failUnlessIn(redir_url, res)
2732 d.addCallback(_check3)
2734 d.addCallback(lambda res:
2735 self.POST(foo_url, t="check", output="JSON"))
2736 def _check_json(res):
2737 data = simplejson.loads(res)
2738 self.failUnlessIn("storage-index", data)
2739 self.failUnless(data["results"]["healthy"])
2740 d.addCallback(_check_json)
2744 def test_POST_DIRURL_check_and_repair(self):
2745 foo_url = self.public_url + "/foo/"
2746 d = self.POST(foo_url, t="check", repair="true")
2748 self.failUnlessIn("Healthy :", res)
2749 d.addCallback(_check)
2750 redir_url = "http://allmydata.org/TARGET"
2751 def _check2(statuscode, target):
2752 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2753 self.failUnlessReallyEqual(target, redir_url)
2754 d.addCallback(lambda res:
2755 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2758 t="check", repair="true",
2759 when_done=redir_url))
2760 d.addCallback(lambda res:
2761 self.POST(foo_url, t="check", return_to=redir_url))
2763 self.failUnlessIn("Healthy :", res)
2764 self.failUnlessIn("Return to file/directory", res)
2765 self.failUnlessIn(redir_url, res)
2766 d.addCallback(_check3)
2769 def test_POST_FILEURL_mdmf_check(self):
2770 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2771 d = self.POST(quux_url, t="check")
2773 self.failUnlessIn("Healthy", res)
2774 d.addCallback(_check)
2775 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2776 d.addCallback(lambda ignored:
2777 self.POST(quux_extension_url, t="check"))
2778 d.addCallback(_check)
2781 def test_POST_FILEURL_mdmf_check_and_repair(self):
2782 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2783 d = self.POST(quux_url, t="check", repair="true")
2785 self.failUnlessIn("Healthy", res)
2786 d.addCallback(_check)
2787 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2788 d.addCallback(lambda ignored:
2789 self.POST(quux_extension_url, t="check", repair="true"))
2790 d.addCallback(_check)
2793 def wait_for_operation(self, ignored, ophandle):
2794 url = "/operations/" + ophandle
2795 url += "?t=status&output=JSON"
2798 data = simplejson.loads(res)
2799 if not data["finished"]:
2800 d = self.stall(delay=1.0)
2801 d.addCallback(self.wait_for_operation, ophandle)
2807 def get_operation_results(self, ignored, ophandle, output=None):
2808 url = "/operations/" + ophandle
2811 url += "&output=" + output
2814 if output and output.lower() == "json":
2815 return simplejson.loads(res)
2820 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2821 d = self.shouldFail2(error.Error,
2822 "test_POST_DIRURL_deepcheck_no_ophandle",
2824 "slow operation requires ophandle=",
2825 self.POST, self.public_url, t="start-deep-check")
2828 def test_POST_DIRURL_deepcheck(self):
2829 def _check_redirect(statuscode, target):
2830 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2831 self.failUnless(target.endswith("/operations/123"))
2832 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2833 self.POST, self.public_url,
2834 t="start-deep-check", ophandle="123")
2835 d.addCallback(self.wait_for_operation, "123")
2836 def _check_json(data):
2837 self.failUnlessReallyEqual(data["finished"], True)
2838 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2839 self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2840 d.addCallback(_check_json)
2841 d.addCallback(self.get_operation_results, "123", "html")
2842 def _check_html(res):
2843 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2844 self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2845 self.failUnlessIn(FAVICON_MARKUP, res)
2846 d.addCallback(_check_html)
2848 d.addCallback(lambda res:
2849 self.GET("/operations/123/"))
2850 d.addCallback(_check_html) # should be the same as without the slash
2852 d.addCallback(lambda res:
2853 self.shouldFail2(error.Error, "one", "404 Not Found",
2854 "No detailed results for SI bogus",
2855 self.GET, "/operations/123/bogus"))
2857 foo_si = self._foo_node.get_storage_index()
2858 foo_si_s = base32.b2a(foo_si)
2859 d.addCallback(lambda res:
2860 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2861 def _check_foo_json(res):
2862 data = simplejson.loads(res)
2863 self.failUnlessEqual(data["storage-index"], foo_si_s)
2864 self.failUnless(data["results"]["healthy"])
2865 d.addCallback(_check_foo_json)
2868 def test_POST_DIRURL_deepcheck_and_repair(self):
2869 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2870 ophandle="124", output="json", followRedirect=True)
2871 d.addCallback(self.wait_for_operation, "124")
2872 def _check_json(data):
2873 self.failUnlessReallyEqual(data["finished"], True)
2874 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2875 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2876 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2877 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2878 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2879 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2880 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2881 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2882 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2883 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2884 d.addCallback(_check_json)
2885 d.addCallback(self.get_operation_results, "124", "html")
2886 def _check_html(res):
2887 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2889 self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2890 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2891 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2893 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2894 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2895 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2897 self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2898 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2899 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2901 self.failUnlessIn(FAVICON_MARKUP, res)
2902 d.addCallback(_check_html)
2905 def test_POST_FILEURL_bad_t(self):
2906 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2907 "POST to file: bad t=bogus",
2908 self.POST, self.public_url + "/foo/bar.txt",
2912 def test_POST_mkdir(self): # return value?
2913 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2914 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2915 d.addCallback(self.failUnlessNodeKeysAre, [])
2918 def test_POST_mkdir_mdmf(self):
2919 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2920 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2921 d.addCallback(lambda node:
2922 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2925 def test_POST_mkdir_sdmf(self):
2926 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2927 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2928 d.addCallback(lambda node:
2929 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2932 def test_POST_mkdir_bad_format(self):
2933 return self.shouldHTTPError("POST_mkdir_bad_format",
2934 400, "Bad Request", "Unknown format: foo",
2935 self.POST, self.public_url +
2936 "/foo?t=mkdir&name=newdir&format=foo")
2938 def test_POST_mkdir_initial_children(self):
2939 (newkids, caps) = self._create_initial_children()
2940 d = self.POST2(self.public_url +
2941 "/foo?t=mkdir-with-children&name=newdir",
2942 simplejson.dumps(newkids))
2943 d.addCallback(lambda res:
2944 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2945 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2946 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2947 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2951 def test_POST_mkdir_initial_children_mdmf(self):
2952 (newkids, caps) = self._create_initial_children()
2953 d = self.POST2(self.public_url +
2954 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2955 simplejson.dumps(newkids))
2956 d.addCallback(lambda res:
2957 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2958 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2959 d.addCallback(lambda node:
2960 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2961 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2962 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2967 def test_POST_mkdir_initial_children_sdmf(self):
2968 (newkids, caps) = self._create_initial_children()
2969 d = self.POST2(self.public_url +
2970 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2971 simplejson.dumps(newkids))
2972 d.addCallback(lambda res:
2973 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2974 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2975 d.addCallback(lambda node:
2976 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2977 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2978 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2982 def test_POST_mkdir_initial_children_bad_format(self):
2983 (newkids, caps) = self._create_initial_children()
2984 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2985 400, "Bad Request", "Unknown format: foo",
2986 self.POST, self.public_url + \
2987 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2988 simplejson.dumps(newkids))
2990 def test_POST_mkdir_immutable(self):
2991 (newkids, caps) = self._create_immutable_children()
2992 d = self.POST2(self.public_url +
2993 "/foo?t=mkdir-immutable&name=newdir",
2994 simplejson.dumps(newkids))
2995 d.addCallback(lambda res:
2996 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2997 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3005 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3006 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3007 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3008 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3011 def test_POST_mkdir_immutable_bad(self):
3012 (newkids, caps) = self._create_initial_children()
3013 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3015 "needed to be immutable but was not",
3018 "/foo?t=mkdir-immutable&name=newdir",
3019 simplejson.dumps(newkids))
3022 def test_POST_mkdir_2(self):
3023 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3024 d.addCallback(lambda res:
3025 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3026 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3027 d.addCallback(self.failUnlessNodeKeysAre, [])
3030 def test_POST_mkdirs_2(self):
3031 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3032 d.addCallback(lambda res:
3033 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3034 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3035 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3036 d.addCallback(self.failUnlessNodeKeysAre, [])
3039 def test_POST_mkdir_no_parentdir_noredirect(self):
3040 d = self.POST("/uri?t=mkdir")
3041 def _after_mkdir(res):
3042 uri.DirectoryURI.init_from_string(res)
3043 d.addCallback(_after_mkdir)
3046 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3047 d = self.POST("/uri?t=mkdir&format=mdmf")
3048 def _after_mkdir(res):
3049 u = uri.from_string(res)
3050 # Check that this is an MDMF writecap
3051 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3052 d.addCallback(_after_mkdir)
3055 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3056 d = self.POST("/uri?t=mkdir&format=sdmf")
3057 def _after_mkdir(res):
3058 u = uri.from_string(res)
3059 self.failUnlessIsInstance(u, uri.DirectoryURI)
3060 d.addCallback(_after_mkdir)
3063 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3064 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3065 400, "Bad Request", "Unknown format: foo",
3066 self.POST, self.public_url +
3067 "/uri?t=mkdir&format=foo")
3069 def test_POST_mkdir_no_parentdir_noredirect2(self):
3070 # make sure form-based arguments (as on the welcome page) still work
3071 d = self.POST("/uri", t="mkdir")
3072 def _after_mkdir(res):
3073 uri.DirectoryURI.init_from_string(res)
3074 d.addCallback(_after_mkdir)
3075 d.addErrback(self.explain_web_error)
3078 def test_POST_mkdir_no_parentdir_redirect(self):
3079 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3080 d.addBoth(self.shouldRedirect, None, statuscode='303')
3081 def _check_target(target):
3082 target = urllib.unquote(target)
3083 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3084 d.addCallback(_check_target)
3087 def test_POST_mkdir_no_parentdir_redirect2(self):
3088 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3089 d.addBoth(self.shouldRedirect, None, statuscode='303')
3090 def _check_target(target):
3091 target = urllib.unquote(target)
3092 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3093 d.addCallback(_check_target)
3094 d.addErrback(self.explain_web_error)
3097 def _make_readonly(self, u):
3098 ro_uri = uri.from_string(u).get_readonly()
3101 return ro_uri.to_string()
3103 def _create_initial_children(self):
3104 contents, n, filecap1 = self.makefile(12)
3105 md1 = {"metakey1": "metavalue1"}
3106 filecap2 = make_mutable_file_uri()
3107 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3108 filecap3 = node3.get_readonly_uri()
3109 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3110 dircap = DirectoryNode(node4, None, None).get_uri()
3111 mdmfcap = make_mutable_file_uri(mdmf=True)
3112 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3113 emptydircap = "URI:DIR2-LIT:"
3114 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3115 "ro_uri": self._make_readonly(filecap1),
3116 "metadata": md1, }],
3117 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3118 "ro_uri": self._make_readonly(filecap2)}],
3119 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3120 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3121 "ro_uri": unknown_rocap}],
3122 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3123 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3124 u"dirchild": ["dirnode", {"rw_uri": dircap,
3125 "ro_uri": self._make_readonly(dircap)}],
3126 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3127 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3128 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3129 "ro_uri": self._make_readonly(mdmfcap)}],
3131 return newkids, {'filecap1': filecap1,
3132 'filecap2': filecap2,
3133 'filecap3': filecap3,
3134 'unknown_rwcap': unknown_rwcap,
3135 'unknown_rocap': unknown_rocap,
3136 'unknown_immcap': unknown_immcap,
3138 'litdircap': litdircap,
3139 'emptydircap': emptydircap,
3142 def _create_immutable_children(self):
3143 contents, n, filecap1 = self.makefile(12)
3144 md1 = {"metakey1": "metavalue1"}
3145 tnode = create_chk_filenode("immutable directory contents\n"*10,
3146 self.get_all_contents())
3147 dnode = DirectoryNode(tnode, None, None)
3148 assert not dnode.is_mutable()
3149 immdircap = dnode.get_uri()
3150 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3151 emptydircap = "URI:DIR2-LIT:"
3152 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3153 "metadata": md1, }],
3154 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3155 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3156 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3157 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3159 return newkids, {'filecap1': filecap1,
3160 'unknown_immcap': unknown_immcap,
3161 'immdircap': immdircap,
3162 'litdircap': litdircap,
3163 'emptydircap': emptydircap}
3165 def test_POST_mkdir_no_parentdir_initial_children(self):
3166 (newkids, caps) = self._create_initial_children()
3167 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3168 def _after_mkdir(res):
3169 self.failUnless(res.startswith("URI:DIR"), res)
3170 n = self.s.create_node_from_uri(res)
3171 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3172 d2.addCallback(lambda ign:
3173 self.failUnlessROChildURIIs(n, u"child-imm",
3175 d2.addCallback(lambda ign:
3176 self.failUnlessRWChildURIIs(n, u"child-mutable",
3178 d2.addCallback(lambda ign:
3179 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3181 d2.addCallback(lambda ign:
3182 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3183 caps['unknown_rwcap']))
3184 d2.addCallback(lambda ign:
3185 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3186 caps['unknown_rocap']))
3187 d2.addCallback(lambda ign:
3188 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3189 caps['unknown_immcap']))
3190 d2.addCallback(lambda ign:
3191 self.failUnlessRWChildURIIs(n, u"dirchild",
3194 d.addCallback(_after_mkdir)
3197 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3198 # the regular /uri?t=mkdir operation is specified to ignore its body.
3199 # Only t=mkdir-with-children pays attention to it.
3200 (newkids, caps) = self._create_initial_children()
3201 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3203 "t=mkdir does not accept children=, "
3204 "try t=mkdir-with-children instead",
3205 self.POST2, "/uri?t=mkdir", # without children
3206 simplejson.dumps(newkids))
3209 def test_POST_noparent_bad(self):
3210 d = self.shouldHTTPError("POST_noparent_bad",
3212 "/uri accepts only PUT, PUT?t=mkdir, "
3213 "POST?t=upload, and POST?t=mkdir",
3214 self.POST, "/uri?t=bogus")
3217 def test_POST_mkdir_no_parentdir_immutable(self):
3218 (newkids, caps) = self._create_immutable_children()
3219 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3220 def _after_mkdir(res):
3221 self.failUnless(res.startswith("URI:DIR"), res)
3222 n = self.s.create_node_from_uri(res)
3223 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3224 d2.addCallback(lambda ign:
3225 self.failUnlessROChildURIIs(n, u"child-imm",
3227 d2.addCallback(lambda ign:
3228 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3229 caps['unknown_immcap']))
3230 d2.addCallback(lambda ign:
3231 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3233 d2.addCallback(lambda ign:
3234 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3236 d2.addCallback(lambda ign:
3237 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3238 caps['emptydircap']))
3240 d.addCallback(_after_mkdir)
3243 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3244 (newkids, caps) = self._create_initial_children()
3245 d = self.shouldFail2(error.Error,
3246 "test_POST_mkdir_no_parentdir_immutable_bad",
3248 "needed to be immutable but was not",
3250 "/uri?t=mkdir-immutable",
3251 simplejson.dumps(newkids))
3254 def test_welcome_page_mkdir_button(self):
3255 # Fetch the welcome page.
3257 def _after_get_welcome_page(res):
3258 MKDIR_BUTTON_RE = re.compile(
3259 '<form action="([^"]*)" method="post".*'
3260 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3261 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3262 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3263 html = res.replace('\n', ' ')
3264 mo = MKDIR_BUTTON_RE.search(html)
3265 self.failUnless(mo, html)
3266 formaction = mo.group(1)
3268 formaname = mo.group(3)
3269 formavalue = mo.group(4)
3270 return (formaction, formt, formaname, formavalue)
3271 d.addCallback(_after_get_welcome_page)
3272 def _after_parse_form(res):
3273 (formaction, formt, formaname, formavalue) = res
3274 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3275 d.addCallback(_after_parse_form)
3276 d.addBoth(self.shouldRedirect, None, statuscode='303')
3279 def test_POST_mkdir_replace(self): # return value?
3280 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3281 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3282 d.addCallback(self.failUnlessNodeKeysAre, [])
3285 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3286 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3287 d.addBoth(self.shouldFail, error.Error,
3288 "POST_mkdir_no_replace_queryarg",
3290 "There was already a child by that name, and you asked me "
3291 "to not replace it")
3292 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3293 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3296 def test_POST_mkdir_no_replace_field(self): # return value?
3297 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3299 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3301 "There was already a child by that name, and you asked me "
3302 "to not replace it")
3303 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3304 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3307 def test_POST_mkdir_whendone_field(self):
3308 d = self.POST(self.public_url + "/foo",
3309 t="mkdir", name="newdir", when_done="/THERE")
3310 d.addBoth(self.shouldRedirect, "/THERE")
3311 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3312 d.addCallback(self.failUnlessNodeKeysAre, [])
3315 def test_POST_mkdir_whendone_queryarg(self):
3316 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3317 t="mkdir", name="newdir")
3318 d.addBoth(self.shouldRedirect, "/THERE")
3319 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3320 d.addCallback(self.failUnlessNodeKeysAre, [])
3323 def test_POST_bad_t(self):
3324 d = self.shouldFail2(error.Error, "POST_bad_t",
3326 "POST to a directory with bad t=BOGUS",
3327 self.POST, self.public_url + "/foo", t="BOGUS")
3330 def test_POST_set_children(self, command_name="set_children"):
3331 contents9, n9, newuri9 = self.makefile(9)
3332 contents10, n10, newuri10 = self.makefile(10)
3333 contents11, n11, newuri11 = self.makefile(11)
3336 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3339 "ctime": 1002777696.7564139,
3340 "mtime": 1002777696.7564139
3343 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3346 "ctime": 1002777696.7564139,
3347 "mtime": 1002777696.7564139
3350 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3353 "ctime": 1002777696.7564139,
3354 "mtime": 1002777696.7564139
3357 }""" % (newuri9, newuri10, newuri11)
3359 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3361 d = client.getPage(url, method="POST", postdata=reqbody)
3363 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3364 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3365 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3367 d.addCallback(_then)
3368 d.addErrback(self.dump_error)
3371 def test_POST_set_children_with_hyphen(self):
3372 return self.test_POST_set_children(command_name="set-children")
3374 def test_POST_link_uri(self):
3375 contents, n, newuri = self.makefile(8)
3376 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3377 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3378 d.addCallback(lambda res:
3379 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3383 def test_POST_link_uri_replace(self):
3384 contents, n, newuri = self.makefile(8)
3385 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3386 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3387 d.addCallback(lambda res:
3388 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3392 def test_POST_link_uri_unknown_bad(self):
3393 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3394 d.addBoth(self.shouldFail, error.Error,
3395 "POST_link_uri_unknown_bad",
3397 "unknown cap in a write slot")
3400 def test_POST_link_uri_unknown_ro_good(self):
3401 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3402 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3405 def test_POST_link_uri_unknown_imm_good(self):
3406 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3407 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3410 def test_POST_link_uri_no_replace_queryarg(self):
3411 contents, n, newuri = self.makefile(8)
3412 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3413 name="bar.txt", uri=newuri)
3414 d.addBoth(self.shouldFail, error.Error,
3415 "POST_link_uri_no_replace_queryarg",
3417 "There was already a child by that name, and you asked me "
3418 "to not replace it")
3419 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3420 d.addCallback(self.failUnlessIsBarDotTxt)
3423 def test_POST_link_uri_no_replace_field(self):
3424 contents, n, newuri = self.makefile(8)
3425 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3426 name="bar.txt", uri=newuri)
3427 d.addBoth(self.shouldFail, error.Error,
3428 "POST_link_uri_no_replace_field",
3430 "There was already a child by that name, and you asked me "
3431 "to not replace it")
3432 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3433 d.addCallback(self.failUnlessIsBarDotTxt)
3436 def test_POST_delete(self, command_name='delete'):
3437 d = self._foo_node.list()
3438 def _check_before(children):
3439 self.failUnlessIn(u"bar.txt", children)
3440 d.addCallback(_check_before)
3441 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3442 d.addCallback(lambda res: self._foo_node.list())
3443 def _check_after(children):
3444 self.failIfIn(u"bar.txt", children)
3445 d.addCallback(_check_after)
3448 def test_POST_unlink(self):
3449 return self.test_POST_delete(command_name='unlink')
3451 def test_POST_rename_file(self):
3452 d = self.POST(self.public_url + "/foo", t="rename",
3453 from_name="bar.txt", to_name='wibble.txt')
3454 d.addCallback(lambda res:
3455 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3456 d.addCallback(lambda res:
3457 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3458 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3459 d.addCallback(self.failUnlessIsBarDotTxt)
3460 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3461 d.addCallback(self.failUnlessIsBarJSON)
3464 def test_POST_rename_file_redundant(self):
3465 d = self.POST(self.public_url + "/foo", t="rename",
3466 from_name="bar.txt", to_name='bar.txt')
3467 d.addCallback(lambda res:
3468 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3469 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3470 d.addCallback(self.failUnlessIsBarDotTxt)
3471 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3472 d.addCallback(self.failUnlessIsBarJSON)
3475 def test_POST_rename_file_replace(self):
3476 # rename a file and replace a directory with it
3477 d = self.POST(self.public_url + "/foo", t="rename",
3478 from_name="bar.txt", to_name='empty')
3479 d.addCallback(lambda res:
3480 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3481 d.addCallback(lambda res:
3482 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3483 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3484 d.addCallback(self.failUnlessIsBarDotTxt)
3485 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3486 d.addCallback(self.failUnlessIsBarJSON)
3489 def test_POST_rename_file_no_replace_queryarg(self):
3490 # rename a file and replace a directory with it
3491 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3492 from_name="bar.txt", to_name='empty')
3493 d.addBoth(self.shouldFail, error.Error,
3494 "POST_rename_file_no_replace_queryarg",
3496 "There was already a child by that name, and you asked me "
3497 "to not replace it")
3498 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3499 d.addCallback(self.failUnlessIsEmptyJSON)
3502 def test_POST_rename_file_no_replace_field(self):
3503 # rename a file and replace a directory with it
3504 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3505 from_name="bar.txt", to_name='empty')
3506 d.addBoth(self.shouldFail, error.Error,
3507 "POST_rename_file_no_replace_field",
3509 "There was already a child by that name, and you asked me "
3510 "to not replace it")
3511 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3512 d.addCallback(self.failUnlessIsEmptyJSON)
3515 def test_POST_rename_file_no_replace_same_link(self):
3516 d = self.POST(self.public_url + "/foo", t="rename",
3517 replace="false", from_name="bar.txt", to_name="bar.txt")
3518 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3519 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3520 d.addCallback(self.failUnlessIsBarDotTxt)
3521 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3522 d.addCallback(self.failUnlessIsBarJSON)
3525 def test_POST_rename_file_replace_only_files(self):
3526 d = self.POST(self.public_url + "/foo", t="rename",
3527 replace="only-files", from_name="bar.txt",
3529 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3530 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3531 d.addCallback(self.failUnlessIsBarDotTxt)
3532 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3533 d.addCallback(self.failUnlessIsBarJSON)
3536 def test_POST_rename_file_replace_only_files_conflict(self):
3537 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3539 "There was already a child by that name, and you asked me to not replace it.",
3540 self.POST, self.public_url + "/foo", t="relink",
3541 replace="only-files", from_name="bar.txt",
3543 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3544 d.addCallback(self.failUnlessIsBarDotTxt)
3545 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3546 d.addCallback(self.failUnlessIsBarJSON)
3549 def failUnlessIsEmptyJSON(self, res):
3550 data = simplejson.loads(res)
3551 self.failUnlessEqual(data[0], "dirnode", data)
3552 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3554 def test_POST_rename_file_to_slash_fail(self):
3555 d = self.POST(self.public_url + "/foo", t="rename",
3556 from_name="bar.txt", to_name='kirk/spock.txt')
3557 d.addBoth(self.shouldFail, error.Error,
3558 "test_POST_rename_file_to_slash_fail",
3560 "to_name= may not contain a slash",
3562 d.addCallback(lambda res:
3563 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3566 def test_POST_rename_file_from_slash_fail(self):
3567 d = self.POST(self.public_url + "/foo", t="rename",
3568 from_name="sub/bar.txt", to_name='spock.txt')
3569 d.addBoth(self.shouldFail, error.Error,
3570 "test_POST_rename_from_file_slash_fail",
3572 "from_name= may not contain a slash",
3574 d.addCallback(lambda res:
3575 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3578 def test_POST_rename_dir(self):
3579 d = self.POST(self.public_url, t="rename",
3580 from_name="foo", to_name='plunk')
3581 d.addCallback(lambda res:
3582 self.failIfNodeHasChild(self.public_root, u"foo"))
3583 d.addCallback(lambda res:
3584 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3585 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3586 d.addCallback(self.failUnlessIsFooJSON)
3589 def test_POST_relink_file(self):
3590 d = self.POST(self.public_url + "/foo", t="relink",
3591 from_name="bar.txt",
3592 to_dir=self.public_root.get_uri() + "/foo/sub")
3593 d.addCallback(lambda res:
3594 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3595 d.addCallback(lambda res:
3596 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3597 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3598 d.addCallback(self.failUnlessIsBarDotTxt)
3599 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3600 d.addCallback(self.failUnlessIsBarJSON)
3603 def test_POST_relink_file_new_name(self):
3604 d = self.POST(self.public_url + "/foo", t="relink",
3605 from_name="bar.txt",
3606 to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3607 d.addCallback(lambda res:
3608 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3609 d.addCallback(lambda res:
3610 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3611 d.addCallback(lambda res:
3612 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3613 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3614 d.addCallback(self.failUnlessIsBarDotTxt)
3615 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3616 d.addCallback(self.failUnlessIsBarJSON)
3619 def test_POST_relink_file_replace(self):
3620 d = self.POST(self.public_url + "/foo", t="relink",
3621 from_name="bar.txt",
3622 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3623 d.addCallback(lambda res:
3624 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3625 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3626 d.addCallback(self.failUnlessIsBarDotTxt)
3627 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3628 d.addCallback(self.failUnlessIsBarJSON)
3631 def test_POST_relink_file_no_replace(self):
3632 d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3634 "There was already a child by that name, and you asked me to not replace it",
3635 self.POST, self.public_url + "/foo", t="relink",
3636 replace="false", from_name="bar.txt",
3637 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3638 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3639 d.addCallback(self.failUnlessIsBarDotTxt)
3640 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3641 d.addCallback(self.failUnlessIsBarJSON)
3642 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3643 d.addCallback(self.failUnlessIsSubBazDotTxt)
3646 def test_POST_relink_file_no_replace_explicitly_same_link(self):
3647 d = self.POST(self.public_url + "/foo", t="relink",
3648 replace="false", from_name="bar.txt",
3649 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3650 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3651 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3652 d.addCallback(self.failUnlessIsBarDotTxt)
3653 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3654 d.addCallback(self.failUnlessIsBarJSON)
3657 def test_POST_relink_file_replace_only_files(self):
3658 d = self.POST(self.public_url + "/foo", t="relink",
3659 replace="only-files", from_name="bar.txt",
3660 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3661 d.addCallback(lambda res:
3662 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3663 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3664 d.addCallback(self.failUnlessIsBarDotTxt)
3665 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3666 d.addCallback(self.failUnlessIsBarJSON)
3669 def test_POST_relink_file_replace_only_files_conflict(self):
3670 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3672 "There was already a child by that name, and you asked me to not replace it.",
3673 self.POST, self.public_url + "/foo", t="relink",
3674 replace="only-files", from_name="bar.txt",
3675 to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3676 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3677 d.addCallback(self.failUnlessIsBarDotTxt)
3678 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3679 d.addCallback(self.failUnlessIsBarJSON)
3682 def test_POST_relink_file_to_slash_fail(self):
3683 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3685 "to_name= may not contain a slash",
3686 self.POST, self.public_url + "/foo", t="relink",
3687 from_name="bar.txt",
3688 to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3689 d.addCallback(lambda res:
3690 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3691 d.addCallback(lambda res:
3692 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3693 d.addCallback(lambda ign:
3694 self.shouldFail2(error.Error,
3695 "test_POST_rename_file_slash_fail2",
3697 "from_name= may not contain a slash",
3698 self.POST, self.public_url + "/foo",
3700 from_name="nope/bar.txt",
3702 to_dir=self.public_root.get_uri() + "/foo/sub"))
3705 def test_POST_relink_file_explicitly_same_link(self):
3706 d = self.POST(self.public_url + "/foo", t="relink",
3707 from_name="bar.txt",
3708 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3709 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3710 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3711 d.addCallback(self.failUnlessIsBarDotTxt)
3712 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3713 d.addCallback(self.failUnlessIsBarJSON)
3716 def test_POST_relink_file_implicitly_same_link(self):
3717 d = self.POST(self.public_url + "/foo", t="relink",
3718 from_name="bar.txt")
3719 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3720 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3721 d.addCallback(self.failUnlessIsBarDotTxt)
3722 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3723 d.addCallback(self.failUnlessIsBarJSON)
3726 def test_POST_relink_file_same_dir(self):
3727 d = self.POST(self.public_url + "/foo", t="relink",
3728 from_name="bar.txt",
3729 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3730 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3731 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3732 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3733 d.addCallback(self.failUnlessIsBarDotTxt)
3734 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3735 d.addCallback(self.failUnlessIsBarJSON)
3738 def test_POST_relink_file_bad_replace(self):
3739 d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3740 "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3742 self.public_url + "/foo", t="relink",
3743 replace="boogabooga", from_name="bar.txt",
3744 to_dir=self.public_root.get_uri() + "/foo/sub")
3747 def test_POST_relink_file_multi_level(self):
3748 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3749 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3750 from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3751 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3752 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3753 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3754 d.addCallback(self.failUnlessIsBarDotTxt)
3755 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3756 d.addCallback(self.failUnlessIsBarJSON)
3759 def test_POST_relink_file_to_uri(self):
3760 d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3761 from_name="bar.txt", to_dir=self._sub_uri)
3762 d.addCallback(lambda res:
3763 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3764 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3765 d.addCallback(self.failUnlessIsBarDotTxt)
3766 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3767 d.addCallback(self.failUnlessIsBarJSON)
3770 def test_POST_relink_file_to_nonexistent_dir(self):
3771 d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3772 "404 Not Found", "No such child: nopechucktesta",
3773 self.POST, self.public_url + "/foo", t="relink",
3774 from_name="bar.txt",
3775 to_dir=self.public_root.get_uri() + "/nopechucktesta")
3778 def test_POST_relink_file_into_file(self):
3779 d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3780 "400 Bad Request", "to_dir is not a directory",
3781 self.POST, self.public_url + "/foo", t="relink",
3782 from_name="bar.txt",
3783 to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3784 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3785 d.addCallback(self.failUnlessIsBazDotTxt)
3786 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3787 d.addCallback(self.failUnlessIsBarDotTxt)
3788 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3789 d.addCallback(self.failUnlessIsBarJSON)
3792 def test_POST_relink_file_to_bad_uri(self):
3793 d = self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3794 "400 Bad Request", "to_dir is not a directory",
3795 self.POST, self.public_url + "/foo", t="relink",
3796 from_name="bar.txt",
3797 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3798 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3799 d.addCallback(self.failUnlessIsBarDotTxt)
3800 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3801 d.addCallback(self.failUnlessIsBarJSON)
3804 def test_POST_relink_dir(self):
3805 d = self.POST(self.public_url + "/foo", t="relink",
3806 from_name="bar.txt",
3807 to_dir=self.public_root.get_uri() + "/foo/empty")
3808 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3809 t="relink", from_name="empty",
3810 to_dir=self.public_root.get_uri() + "/foo/sub"))
3811 d.addCallback(lambda res:
3812 self.failIfNodeHasChild(self._foo_node, u"empty"))
3813 d.addCallback(lambda res:
3814 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3815 d.addCallback(lambda res:
3816 self._sub_node.get_child_at_path(u"empty"))
3817 d.addCallback(lambda node:
3818 self.failUnlessNodeHasChild(node, u"bar.txt"))
3819 d.addCallback(lambda res:
3820 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3821 d.addCallback(self.failUnlessIsBarDotTxt)
3824 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3825 """ If target is not None then the redirection has to go to target. If
3826 statuscode is not None then the redirection has to be accomplished with
3827 that HTTP status code."""
3828 if not isinstance(res, failure.Failure):
3829 to_where = (target is None) and "somewhere" or ("to " + target)
3830 self.fail("%s: we were expecting to get redirected %s, not get an"
3831 " actual page: %s" % (which, to_where, res))
3832 res.trap(error.PageRedirect)
3833 if statuscode is not None:
3834 self.failUnlessReallyEqual(res.value.status, statuscode,
3835 "%s: not a redirect" % which)
3836 if target is not None:
3837 # the PageRedirect does not seem to capture the uri= query arg
3838 # properly, so we can't check for it.
3839 realtarget = self.webish_url + target
3840 self.failUnlessReallyEqual(res.value.location, realtarget,
3841 "%s: wrong target" % which)
3842 return res.value.location
3844 def test_GET_URI_form(self):
3845 base = "/uri?uri=%s" % self._bar_txt_uri
3846 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3847 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3849 d.addBoth(self.shouldRedirect, targetbase)
3850 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3851 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3852 d.addCallback(lambda res: self.GET(base+"&t=json"))
3853 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3854 d.addCallback(self.log, "about to get file by uri")
3855 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3856 d.addCallback(self.failUnlessIsBarDotTxt)
3857 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3858 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3859 followRedirect=True))
3860 d.addCallback(self.failUnlessIsFooJSON)
3861 d.addCallback(self.log, "got dir by uri")
3865 def test_GET_URI_form_bad(self):
3866 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3867 "400 Bad Request", "GET /uri requires uri=",
3871 def test_GET_rename_form(self):
3872 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3873 followRedirect=True)
3875 self.failUnlessIn('name="when_done" value="."', res)
3876 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3877 self.failUnlessIn(FAVICON_MARKUP, res)
3878 d.addCallback(_check)
3881 def log(self, res, msg):
3882 #print "MSG: %s RES: %s" % (msg, res)
3886 def test_GET_URI_URL(self):
3887 base = "/uri/%s" % self._bar_txt_uri
3889 d.addCallback(self.failUnlessIsBarDotTxt)
3890 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3891 d.addCallback(self.failUnlessIsBarDotTxt)
3892 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3893 d.addCallback(self.failUnlessIsBarDotTxt)
3896 def test_GET_URI_URL_dir(self):
3897 base = "/uri/%s?t=json" % self._foo_uri
3899 d.addCallback(self.failUnlessIsFooJSON)
3902 def test_GET_URI_URL_missing(self):
3903 base = "/uri/%s" % self._bad_file_uri
3904 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3905 http.GONE, None, "NotEnoughSharesError",
3907 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3908 # here? we must arrange for a download to fail after target.open()
3909 # has been called, and then inspect the response to see that it is
3910 # shorter than we expected.
3913 def test_PUT_DIRURL_uri(self):
3914 d = self.s.create_dirnode()
3916 new_uri = dn.get_uri()
3917 # replace /foo with a new (empty) directory
3918 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3919 d.addCallback(lambda res:
3920 self.failUnlessReallyEqual(res.strip(), new_uri))
3921 d.addCallback(lambda res:
3922 self.failUnlessRWChildURIIs(self.public_root,
3926 d.addCallback(_made_dir)
3929 def test_PUT_DIRURL_uri_noreplace(self):
3930 d = self.s.create_dirnode()
3932 new_uri = dn.get_uri()
3933 # replace /foo with a new (empty) directory, but ask that
3934 # replace=false, so it should fail
3935 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3936 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3938 self.public_url + "/foo?t=uri&replace=false",
3940 d.addCallback(lambda res:
3941 self.failUnlessRWChildURIIs(self.public_root,
3945 d.addCallback(_made_dir)
3948 def test_PUT_DIRURL_bad_t(self):
3949 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3950 "400 Bad Request", "PUT to a directory",
3951 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3952 d.addCallback(lambda res:
3953 self.failUnlessRWChildURIIs(self.public_root,
3958 def test_PUT_NEWFILEURL_uri(self):
3959 contents, n, new_uri = self.makefile(8)
3960 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3961 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3962 d.addCallback(lambda res:
3963 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3967 def test_PUT_NEWFILEURL_mdmf(self):
3968 new_contents = self.NEWFILE_CONTENTS * 300000
3969 d = self.PUT(self.public_url + \
3970 "/foo/mdmf.txt?format=mdmf",
3972 d.addCallback(lambda ignored:
3973 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3974 def _got_json(json):
3975 data = simplejson.loads(json)
3977 self.failUnlessIn("format", data)
3978 self.failUnlessEqual(data["format"], "MDMF")
3979 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3980 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3981 d.addCallback(_got_json)
3984 def test_PUT_NEWFILEURL_sdmf(self):
3985 new_contents = self.NEWFILE_CONTENTS * 300000
3986 d = self.PUT(self.public_url + \
3987 "/foo/sdmf.txt?format=sdmf",
3989 d.addCallback(lambda ignored:
3990 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3991 def _got_json(json):
3992 data = simplejson.loads(json)
3994 self.failUnlessIn("format", data)
3995 self.failUnlessEqual(data["format"], "SDMF")
3996 d.addCallback(_got_json)
3999 def test_PUT_NEWFILEURL_bad_format(self):
4000 new_contents = self.NEWFILE_CONTENTS * 300000
4001 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4002 400, "Bad Request", "Unknown format: foo",
4003 self.PUT, self.public_url + \
4004 "/foo/foo.txt?format=foo",
4007 def test_PUT_NEWFILEURL_uri_replace(self):
4008 contents, n, new_uri = self.makefile(8)
4009 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4010 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4011 d.addCallback(lambda res:
4012 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4016 def test_PUT_NEWFILEURL_uri_no_replace(self):
4017 contents, n, new_uri = self.makefile(8)
4018 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4019 d.addBoth(self.shouldFail, error.Error,
4020 "PUT_NEWFILEURL_uri_no_replace",
4022 "There was already a child by that name, and you asked me "
4023 "to not replace it")
4026 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4027 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4028 d.addBoth(self.shouldFail, error.Error,
4029 "POST_put_uri_unknown_bad",
4031 "unknown cap in a write slot")
4034 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4035 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4036 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4037 u"put-future-ro.txt")
4040 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4041 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4042 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4043 u"put-future-imm.txt")
4046 def test_PUT_NEWFILE_URI(self):
4047 file_contents = "New file contents here\n"
4048 d = self.PUT("/uri", file_contents)
4050 assert isinstance(uri, str), uri
4051 self.failUnlessIn(uri, self.get_all_contents())
4052 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4054 return self.GET("/uri/%s" % uri)
4055 d.addCallback(_check)
4057 self.failUnlessReallyEqual(res, file_contents)
4058 d.addCallback(_check2)
4061 def test_PUT_NEWFILE_URI_not_mutable(self):
4062 file_contents = "New file contents here\n"
4063 d = self.PUT("/uri?mutable=false", file_contents)
4065 assert isinstance(uri, str), uri
4066 self.failUnlessIn(uri, self.get_all_contents())
4067 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4069 return self.GET("/uri/%s" % uri)
4070 d.addCallback(_check)
4072 self.failUnlessReallyEqual(res, file_contents)
4073 d.addCallback(_check2)
4076 def test_PUT_NEWFILE_URI_only_PUT(self):
4077 d = self.PUT("/uri?t=bogus", "")
4078 d.addBoth(self.shouldFail, error.Error,
4079 "PUT_NEWFILE_URI_only_PUT",
4081 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4084 def test_PUT_NEWFILE_URI_mutable(self):
4085 file_contents = "New file contents here\n"
4086 d = self.PUT("/uri?mutable=true", file_contents)
4087 def _check1(filecap):
4088 filecap = filecap.strip()
4089 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4090 self.filecap = filecap
4091 u = uri.WriteableSSKFileURI.init_from_string(filecap)
4092 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4093 n = self.s.create_node_from_uri(filecap)
4094 return n.download_best_version()
4095 d.addCallback(_check1)
4097 self.failUnlessReallyEqual(data, file_contents)
4098 return self.GET("/uri/%s" % urllib.quote(self.filecap))
4099 d.addCallback(_check2)
4101 self.failUnlessReallyEqual(res, file_contents)
4102 d.addCallback(_check3)
4105 def test_PUT_mkdir(self):
4106 d = self.PUT("/uri?t=mkdir", "")
4108 n = self.s.create_node_from_uri(uri.strip())
4109 d2 = self.failUnlessNodeKeysAre(n, [])
4110 d2.addCallback(lambda res:
4111 self.GET("/uri/%s?t=json" % uri))
4113 d.addCallback(_check)
4114 d.addCallback(self.failUnlessIsEmptyJSON)
4117 def test_PUT_mkdir_mdmf(self):
4118 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4120 u = uri.from_string(res)
4121 # Check that this is an MDMF writecap
4122 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4126 def test_PUT_mkdir_sdmf(self):
4127 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4129 u = uri.from_string(res)
4130 self.failUnlessIsInstance(u, uri.DirectoryURI)
4134 def test_PUT_mkdir_bad_format(self):
4135 return self.shouldHTTPError("PUT_mkdir_bad_format",
4136 400, "Bad Request", "Unknown format: foo",
4137 self.PUT, "/uri?t=mkdir&format=foo",
4140 def test_POST_check(self):
4141 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4143 # this returns a string form of the results, which are probably
4144 # None since we're using fake filenodes.
4145 # TODO: verify that the check actually happened, by changing
4146 # FakeCHKFileNode to count how many times .check() has been
4149 d.addCallback(_done)
4153 def test_PUT_update_at_offset(self):
4154 file_contents = "test file" * 100000 # about 900 KiB
4155 d = self.PUT("/uri?mutable=true", file_contents)
4157 self.filecap = filecap
4158 new_data = file_contents[:100]
4159 new = "replaced and so on"
4161 new_data += file_contents[len(new_data):]
4162 assert len(new_data) == len(file_contents)
4163 self.new_data = new_data
4164 d.addCallback(_then)
4165 d.addCallback(lambda ignored:
4166 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4167 "replaced and so on"))
4168 def _get_data(filecap):
4169 n = self.s.create_node_from_uri(filecap)
4170 return n.download_best_version()
4171 d.addCallback(_get_data)
4172 d.addCallback(lambda results:
4173 self.failUnlessEqual(results, self.new_data))
4174 # Now try appending things to the file
4175 d.addCallback(lambda ignored:
4176 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4178 d.addCallback(_get_data)
4179 d.addCallback(lambda results:
4180 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4181 # and try replacing the beginning of the file
4182 d.addCallback(lambda ignored:
4183 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4184 d.addCallback(_get_data)
4185 d.addCallback(lambda results:
4186 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4189 def test_PUT_update_at_invalid_offset(self):
4190 file_contents = "test file" * 100000 # about 900 KiB
4191 d = self.PUT("/uri?mutable=true", file_contents)
4193 self.filecap = filecap
4194 d.addCallback(_then)
4195 # Negative offsets should cause an error.
4196 d.addCallback(lambda ignored:
4197 self.shouldHTTPError("PUT_update_at_invalid_offset",
4201 "/uri/%s?offset=-1" % self.filecap,
4205 def test_PUT_update_at_offset_immutable(self):
4206 file_contents = "Test file" * 100000
4207 d = self.PUT("/uri", file_contents)
4209 self.filecap = filecap
4210 d.addCallback(_then)
4211 d.addCallback(lambda ignored:
4212 self.shouldHTTPError("PUT_update_at_offset_immutable",
4216 "/uri/%s?offset=50" % self.filecap,
4221 def test_bad_method(self):
4222 url = self.webish_url + self.public_url + "/foo/bar.txt"
4223 d = self.shouldHTTPError("bad_method",
4224 501, "Not Implemented",
4225 "I don't know how to treat a BOGUS request.",
4226 client.getPage, url, method="BOGUS")
4229 def test_short_url(self):
4230 url = self.webish_url + "/uri"
4231 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4232 "I don't know how to treat a DELETE request.",
4233 client.getPage, url, method="DELETE")
4236 def test_ophandle_bad(self):
4237 url = self.webish_url + "/operations/bogus?t=status"
4238 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4239 "unknown/expired handle 'bogus'",
4240 client.getPage, url)
4243 def test_ophandle_cancel(self):
4244 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4245 followRedirect=True)
4246 d.addCallback(lambda ignored:
4247 self.GET("/operations/128?t=status&output=JSON"))
4249 data = simplejson.loads(res)
4250 self.failUnless("finished" in data, res)
4251 monitor = self.ws.root.child_operations.handles["128"][0]
4252 d = self.POST("/operations/128?t=cancel&output=JSON")
4254 data = simplejson.loads(res)
4255 self.failUnless("finished" in data, res)
4256 # t=cancel causes the handle to be forgotten
4257 self.failUnless(monitor.is_cancelled())
4258 d.addCallback(_check2)
4260 d.addCallback(_check1)
4261 d.addCallback(lambda ignored:
4262 self.shouldHTTPError("ophandle_cancel",
4263 404, "404 Not Found",
4264 "unknown/expired handle '128'",
4266 "/operations/128?t=status&output=JSON"))
4269 def test_ophandle_retainfor(self):
4270 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4271 followRedirect=True)
4272 d.addCallback(lambda ignored:
4273 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4275 data = simplejson.loads(res)
4276 self.failUnless("finished" in data, res)
4277 d.addCallback(_check1)
4278 # the retain-for=0 will cause the handle to be expired very soon
4279 d.addCallback(lambda ign:
4280 self.clock.advance(2.0))
4281 d.addCallback(lambda ignored:
4282 self.shouldHTTPError("ophandle_retainfor",
4283 404, "404 Not Found",
4284 "unknown/expired handle '129'",
4286 "/operations/129?t=status&output=JSON"))
4289 def test_ophandle_release_after_complete(self):
4290 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4291 followRedirect=True)
4292 d.addCallback(self.wait_for_operation, "130")
4293 d.addCallback(lambda ignored:
4294 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4295 # the release-after-complete=true will cause the handle to be expired
4296 d.addCallback(lambda ignored:
4297 self.shouldHTTPError("ophandle_release_after_complete",
4298 404, "404 Not Found",
4299 "unknown/expired handle '130'",
4301 "/operations/130?t=status&output=JSON"))
4304 def test_uncollected_ophandle_expiration(self):
4305 # uncollected ophandles should expire after 4 days
4306 def _make_uncollected_ophandle(ophandle):
4307 d = self.POST(self.public_url +
4308 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4309 followRedirect=False)
4310 # When we start the operation, the webapi server will want
4311 # to redirect us to the page for the ophandle, so we get
4312 # confirmation that the operation has started. If the
4313 # manifest operation has finished by the time we get there,
4314 # following that redirect (by setting followRedirect=True
4315 # above) has the side effect of collecting the ophandle that
4316 # we've just created, which means that we can't use the
4317 # ophandle to test the uncollected timeout anymore. So,
4318 # instead, catch the 302 here and don't follow it.
4319 d.addBoth(self.should302, "uncollected_ophandle_creation")
4321 # Create an ophandle, don't collect it, then advance the clock by
4322 # 4 days - 1 second and make sure that the ophandle is still there.
4323 d = _make_uncollected_ophandle(131)
4324 d.addCallback(lambda ign:
4325 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4326 d.addCallback(lambda ign:
4327 self.GET("/operations/131?t=status&output=JSON"))
4329 data = simplejson.loads(res)
4330 self.failUnless("finished" in data, res)
4331 d.addCallback(_check1)
4332 # Create an ophandle, don't collect it, then try to collect it
4333 # after 4 days. It should be gone.
4334 d.addCallback(lambda ign:
4335 _make_uncollected_ophandle(132))
4336 d.addCallback(lambda ign:
4337 self.clock.advance(96*60*60))
4338 d.addCallback(lambda ign:
4339 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4340 404, "404 Not Found",
4341 "unknown/expired handle '132'",
4343 "/operations/132?t=status&output=JSON"))
4346 def test_collected_ophandle_expiration(self):
4347 # collected ophandles should expire after 1 day
4348 def _make_collected_ophandle(ophandle):
4349 d = self.POST(self.public_url +
4350 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4351 followRedirect=True)
4352 # By following the initial redirect, we collect the ophandle
4353 # we've just created.
4355 # Create a collected ophandle, then collect it after 23 hours
4356 # and 59 seconds to make sure that it is still there.
4357 d = _make_collected_ophandle(133)
4358 d.addCallback(lambda ign:
4359 self.clock.advance((24*60*60) - 1))
4360 d.addCallback(lambda ign:
4361 self.GET("/operations/133?t=status&output=JSON"))
4363 data = simplejson.loads(res)
4364 self.failUnless("finished" in data, res)
4365 d.addCallback(_check1)
4366 # Create another uncollected ophandle, then try to collect it
4367 # after 24 hours to make sure that it is gone.
4368 d.addCallback(lambda ign:
4369 _make_collected_ophandle(134))
4370 d.addCallback(lambda ign:
4371 self.clock.advance(24*60*60))
4372 d.addCallback(lambda ign:
4373 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4374 404, "404 Not Found",
4375 "unknown/expired handle '134'",
4377 "/operations/134?t=status&output=JSON"))
4380 def test_incident(self):
4381 d = self.POST("/report_incident", details="eek")
4383 self.failIfIn("<html>", res)
4384 self.failUnlessIn("An incident report has been saved", res)
4385 d.addCallback(_done)
4388 def test_static(self):
4389 webdir = os.path.join(self.staticdir, "subdir")
4390 fileutil.make_dirs(webdir)
4391 f = open(os.path.join(webdir, "hello.txt"), "wb")
4395 d = self.GET("/static/subdir/hello.txt")
4397 self.failUnlessReallyEqual(res, "hello")
4398 d.addCallback(_check)
4402 class IntroducerWeb(unittest.TestCase):
4407 d = defer.succeed(None)
4409 d.addCallback(lambda ign: self.node.stopService())
4410 d.addCallback(flushEventualQueue)
4413 def test_welcome(self):
4414 basedir = "web.IntroducerWeb.test_welcome"
4416 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4417 self.node = IntroducerNode(basedir)
4418 self.ws = self.node.getServiceNamed("webish")
4420 d = fireEventually(None)
4421 d.addCallback(lambda ign: self.node.startService())
4422 d.addCallback(lambda ign: self.node.when_tub_ready())
4424 d.addCallback(lambda ign: self.GET("/"))
4426 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4427 self.failUnlessIn(FAVICON_MARKUP, res)
4428 self.failUnlessIn('Page rendered at', res)
4429 self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4430 d.addCallback(_check)
4433 def GET(self, urlpath, followRedirect=False, return_response=False,
4435 # if return_response=True, this fires with (data, statuscode,
4436 # respheaders) instead of just data.
4437 assert not isinstance(urlpath, unicode)
4438 url = self.ws.getURL().rstrip('/') + urlpath
4439 factory = HTTPClientGETFactory(url, method="GET",
4440 followRedirect=followRedirect, **kwargs)
4441 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4442 d = factory.deferred
4443 def _got_data(data):
4444 return (data, factory.status, factory.response_headers)
4446 d.addCallback(_got_data)
4447 return factory.deferred
4450 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4451 def test_load_file(self):
4452 # This will raise an exception unless a well-formed XML file is found under that name.
4453 common.getxmlfile('directory.xhtml').load()
4455 def test_parse_replace_arg(self):
4456 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4457 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4458 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4460 self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4462 def test_abbreviate_time(self):
4463 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4464 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4465 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4466 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4467 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4468 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4470 def test_compute_rate(self):
4471 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4472 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4473 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4474 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4475 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4476 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4477 self.shouldFail(AssertionError, "test_compute_rate", "",
4478 common.compute_rate, -100, 10)
4479 self.shouldFail(AssertionError, "test_compute_rate", "",
4480 common.compute_rate, 100, -10)
4483 rate = common.compute_rate(10*1000*1000, 1)
4484 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4486 def test_abbreviate_rate(self):
4487 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4488 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4489 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4490 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4492 def test_abbreviate_size(self):
4493 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4494 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4495 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4496 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4497 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4499 def test_plural(self):
4501 return "%d second%s" % (s, status.plural(s))
4502 self.failUnlessReallyEqual(convert(0), "0 seconds")
4503 self.failUnlessReallyEqual(convert(1), "1 second")
4504 self.failUnlessReallyEqual(convert(2), "2 seconds")
4506 return "has share%s: %s" % (status.plural(s), ",".join(s))
4507 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4508 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4509 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4512 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4514 def CHECK(self, ign, which, args, clientnum=0):
4515 fileurl = self.fileurls[which]
4516 url = fileurl + "?" + args
4517 return self.GET(url, method="POST", clientnum=clientnum)
4519 def test_filecheck(self):
4520 self.basedir = "web/Grid/filecheck"
4522 c0 = self.g.clients[0]
4525 d = c0.upload(upload.Data(DATA, convergence=""))
4526 def _stash_uri(ur, which):
4527 self.uris[which] = ur.get_uri()
4528 d.addCallback(_stash_uri, "good")
4529 d.addCallback(lambda ign:
4530 c0.upload(upload.Data(DATA+"1", convergence="")))
4531 d.addCallback(_stash_uri, "sick")
4532 d.addCallback(lambda ign:
4533 c0.upload(upload.Data(DATA+"2", convergence="")))
4534 d.addCallback(_stash_uri, "dead")
4535 def _stash_mutable_uri(n, which):
4536 self.uris[which] = n.get_uri()
4537 assert isinstance(self.uris[which], str)
4538 d.addCallback(lambda ign:
4539 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4540 d.addCallback(_stash_mutable_uri, "corrupt")
4541 d.addCallback(lambda ign:
4542 c0.upload(upload.Data("literal", convergence="")))
4543 d.addCallback(_stash_uri, "small")
4544 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4545 d.addCallback(_stash_mutable_uri, "smalldir")
4547 def _compute_fileurls(ignored):
4549 for which in self.uris:
4550 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4551 d.addCallback(_compute_fileurls)
4553 def _clobber_shares(ignored):
4554 good_shares = self.find_uri_shares(self.uris["good"])
4555 self.failUnlessReallyEqual(len(good_shares), 10)
4556 sick_shares = self.find_uri_shares(self.uris["sick"])
4557 os.unlink(sick_shares[0][2])
4558 dead_shares = self.find_uri_shares(self.uris["dead"])
4559 for i in range(1, 10):
4560 os.unlink(dead_shares[i][2])
4561 c_shares = self.find_uri_shares(self.uris["corrupt"])
4562 cso = CorruptShareOptions()
4563 cso.stdout = StringIO()
4564 cso.parseOptions([c_shares[0][2]])
4566 d.addCallback(_clobber_shares)
4568 d.addCallback(self.CHECK, "good", "t=check")
4569 def _got_html_good(res):
4570 self.failUnlessIn("Healthy", res)
4571 self.failIfIn("Not Healthy", res)
4572 self.failUnlessIn(FAVICON_MARKUP, res)
4573 d.addCallback(_got_html_good)
4574 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4575 def _got_html_good_return_to(res):
4576 self.failUnlessIn("Healthy", res)
4577 self.failIfIn("Not Healthy", res)
4578 self.failUnlessIn('<a href="somewhere">Return to file', res)
4579 d.addCallback(_got_html_good_return_to)
4580 d.addCallback(self.CHECK, "good", "t=check&output=json")
4581 def _got_json_good(res):
4582 r = simplejson.loads(res)
4583 self.failUnlessEqual(r["summary"], "Healthy")
4584 self.failUnless(r["results"]["healthy"])
4585 self.failIfIn("needs-rebalancing", r["results"])
4586 self.failUnless(r["results"]["recoverable"])
4587 d.addCallback(_got_json_good)
4589 d.addCallback(self.CHECK, "small", "t=check")
4590 def _got_html_small(res):
4591 self.failUnlessIn("Literal files are always healthy", res)
4592 self.failIfIn("Not Healthy", res)
4593 d.addCallback(_got_html_small)
4594 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4595 def _got_html_small_return_to(res):
4596 self.failUnlessIn("Literal files are always healthy", res)
4597 self.failIfIn("Not Healthy", res)
4598 self.failUnlessIn('<a href="somewhere">Return to file', res)
4599 d.addCallback(_got_html_small_return_to)
4600 d.addCallback(self.CHECK, "small", "t=check&output=json")
4601 def _got_json_small(res):
4602 r = simplejson.loads(res)
4603 self.failUnlessEqual(r["storage-index"], "")
4604 self.failUnless(r["results"]["healthy"])
4605 d.addCallback(_got_json_small)
4607 d.addCallback(self.CHECK, "smalldir", "t=check")
4608 def _got_html_smalldir(res):
4609 self.failUnlessIn("Literal files are always healthy", res)
4610 self.failIfIn("Not Healthy", res)
4611 d.addCallback(_got_html_smalldir)
4612 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4613 def _got_json_smalldir(res):
4614 r = simplejson.loads(res)
4615 self.failUnlessEqual(r["storage-index"], "")
4616 self.failUnless(r["results"]["healthy"])
4617 d.addCallback(_got_json_smalldir)
4619 d.addCallback(self.CHECK, "sick", "t=check")
4620 def _got_html_sick(res):
4621 self.failUnlessIn("Not Healthy", res)
4622 d.addCallback(_got_html_sick)
4623 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4624 def _got_json_sick(res):
4625 r = simplejson.loads(res)
4626 self.failUnlessEqual(r["summary"],
4627 "Not Healthy: 9 shares (enc 3-of-10)")
4628 self.failIf(r["results"]["healthy"])
4629 self.failUnless(r["results"]["recoverable"])
4630 self.failIfIn("needs-rebalancing", r["results"])
4631 d.addCallback(_got_json_sick)
4633 d.addCallback(self.CHECK, "dead", "t=check")
4634 def _got_html_dead(res):
4635 self.failUnlessIn("Not Healthy", res)
4636 d.addCallback(_got_html_dead)
4637 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4638 def _got_json_dead(res):
4639 r = simplejson.loads(res)
4640 self.failUnlessEqual(r["summary"],
4641 "Not Healthy: 1 shares (enc 3-of-10)")
4642 self.failIf(r["results"]["healthy"])
4643 self.failIf(r["results"]["recoverable"])
4644 self.failIfIn("needs-rebalancing", r["results"])
4645 d.addCallback(_got_json_dead)
4647 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4648 def _got_html_corrupt(res):
4649 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4650 d.addCallback(_got_html_corrupt)
4651 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4652 def _got_json_corrupt(res):
4653 r = simplejson.loads(res)
4654 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4655 self.failIf(r["results"]["healthy"])
4656 self.failUnless(r["results"]["recoverable"])
4657 self.failIfIn("needs-rebalancing", r["results"])
4658 self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4659 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4660 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4661 d.addCallback(_got_json_corrupt)
4663 d.addErrback(self.explain_web_error)
4666 def test_repair_html(self):
4667 self.basedir = "web/Grid/repair_html"
4669 c0 = self.g.clients[0]
4672 d = c0.upload(upload.Data(DATA, convergence=""))
4673 def _stash_uri(ur, which):
4674 self.uris[which] = ur.get_uri()
4675 d.addCallback(_stash_uri, "good")
4676 d.addCallback(lambda ign:
4677 c0.upload(upload.Data(DATA+"1", convergence="")))
4678 d.addCallback(_stash_uri, "sick")
4679 d.addCallback(lambda ign:
4680 c0.upload(upload.Data(DATA+"2", convergence="")))
4681 d.addCallback(_stash_uri, "dead")
4682 def _stash_mutable_uri(n, which):
4683 self.uris[which] = n.get_uri()
4684 assert isinstance(self.uris[which], str)
4685 d.addCallback(lambda ign:
4686 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4687 d.addCallback(_stash_mutable_uri, "corrupt")
4689 def _compute_fileurls(ignored):
4691 for which in self.uris:
4692 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4693 d.addCallback(_compute_fileurls)
4695 def _clobber_shares(ignored):
4696 good_shares = self.find_uri_shares(self.uris["good"])
4697 self.failUnlessReallyEqual(len(good_shares), 10)
4698 sick_shares = self.find_uri_shares(self.uris["sick"])
4699 os.unlink(sick_shares[0][2])
4700 dead_shares = self.find_uri_shares(self.uris["dead"])
4701 for i in range(1, 10):
4702 os.unlink(dead_shares[i][2])
4703 c_shares = self.find_uri_shares(self.uris["corrupt"])
4704 cso = CorruptShareOptions()
4705 cso.stdout = StringIO()
4706 cso.parseOptions([c_shares[0][2]])
4708 d.addCallback(_clobber_shares)
4710 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4711 def _got_html_good(res):
4712 self.failUnlessIn("Healthy", res)
4713 self.failIfIn("Not Healthy", res)
4714 self.failUnlessIn("No repair necessary", res)
4715 self.failUnlessIn(FAVICON_MARKUP, res)
4716 d.addCallback(_got_html_good)
4718 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4719 def _got_html_sick(res):
4720 self.failUnlessIn("Healthy : healthy", res)
4721 self.failIfIn("Not Healthy", res)
4722 self.failUnlessIn("Repair successful", res)
4723 d.addCallback(_got_html_sick)
4725 # repair of a dead file will fail, of course, but it isn't yet
4726 # clear how this should be reported. Right now it shows up as
4729 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4730 #def _got_html_dead(res):
4732 # self.failUnlessIn("Healthy : healthy", res)
4733 # self.failIfIn("Not Healthy", res)
4734 # self.failUnlessIn("No repair necessary", res)
4735 #d.addCallback(_got_html_dead)
4737 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4738 def _got_html_corrupt(res):
4739 self.failUnlessIn("Healthy : Healthy", res)
4740 self.failIfIn("Not Healthy", res)
4741 self.failUnlessIn("Repair successful", res)
4742 d.addCallback(_got_html_corrupt)
4744 d.addErrback(self.explain_web_error)
4747 def test_repair_json(self):
4748 self.basedir = "web/Grid/repair_json"
4750 c0 = self.g.clients[0]
4753 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4754 def _stash_uri(ur, which):
4755 self.uris[which] = ur.get_uri()
4756 d.addCallback(_stash_uri, "sick")
4758 def _compute_fileurls(ignored):
4760 for which in self.uris:
4761 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4762 d.addCallback(_compute_fileurls)
4764 def _clobber_shares(ignored):
4765 sick_shares = self.find_uri_shares(self.uris["sick"])
4766 os.unlink(sick_shares[0][2])
4767 d.addCallback(_clobber_shares)
4769 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4770 def _got_json_sick(res):
4771 r = simplejson.loads(res)
4772 self.failUnlessReallyEqual(r["repair-attempted"], True)
4773 self.failUnlessReallyEqual(r["repair-successful"], True)
4774 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4775 "Not Healthy: 9 shares (enc 3-of-10)")
4776 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4777 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4778 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4779 d.addCallback(_got_json_sick)
4781 d.addErrback(self.explain_web_error)
4784 def test_unknown(self, immutable=False):
4785 self.basedir = "web/Grid/unknown"
4787 self.basedir = "web/Grid/unknown-immutable"
4790 c0 = self.g.clients[0]
4794 # the future cap format may contain slashes, which must be tolerated
4795 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4799 name = u"future-imm"
4800 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4801 d = c0.create_immutable_dirnode({name: (future_node, {})})
4804 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4805 d = c0.create_dirnode()
4807 def _stash_root_and_create_file(n):
4809 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4810 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4812 return self.rootnode.set_node(name, future_node)
4813 d.addCallback(_stash_root_and_create_file)
4815 # make sure directory listing tolerates unknown nodes
4816 d.addCallback(lambda ign: self.GET(self.rooturl))
4817 def _check_directory_html(res, expected_type_suffix):
4818 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4819 '<td>%s</td>' % (expected_type_suffix, str(name)),
4821 self.failUnless(re.search(pattern, res), res)
4822 # find the More Info link for name, should be relative
4823 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4824 info_url = mo.group(1)
4825 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4827 d.addCallback(_check_directory_html, "-IMM")
4829 d.addCallback(_check_directory_html, "")
4831 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4832 def _check_directory_json(res, expect_rw_uri):
4833 data = simplejson.loads(res)
4834 self.failUnlessEqual(data[0], "dirnode")
4835 f = data[1]["children"][name]
4836 self.failUnlessEqual(f[0], "unknown")
4838 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4840 self.failIfIn("rw_uri", f[1])
4842 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4844 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4845 self.failUnlessIn("metadata", f[1])
4846 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4848 def _check_info(res, expect_rw_uri, expect_ro_uri):
4849 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4851 self.failUnlessIn(unknown_rwcap, res)
4854 self.failUnlessIn(unknown_immcap, res)
4856 self.failUnlessIn(unknown_rocap, res)
4858 self.failIfIn(unknown_rocap, res)
4859 self.failIfIn("Raw data as", res)
4860 self.failIfIn("Directory writecap", res)
4861 self.failIfIn("Checker Operations", res)
4862 self.failIfIn("Mutable File Operations", res)
4863 self.failIfIn("Directory Operations", res)
4865 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4866 # why they fail. Possibly related to ticket #922.
4868 d.addCallback(lambda ign: self.GET(expected_info_url))
4869 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4870 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4871 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4873 def _check_json(res, expect_rw_uri):
4874 data = simplejson.loads(res)
4875 self.failUnlessEqual(data[0], "unknown")
4877 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4879 self.failIfIn("rw_uri", data[1])
4882 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4883 self.failUnlessReallyEqual(data[1]["mutable"], False)
4885 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4886 self.failUnlessReallyEqual(data[1]["mutable"], True)
4888 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4889 self.failIfIn("mutable", data[1])
4891 # TODO: check metadata contents
4892 self.failUnlessIn("metadata", data[1])
4894 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4895 d.addCallback(_check_json, expect_rw_uri=not immutable)
4897 # and make sure that a read-only version of the directory can be
4898 # rendered too. This version will not have unknown_rwcap, whether
4899 # or not future_node was immutable.
4900 d.addCallback(lambda ign: self.GET(self.rourl))
4902 d.addCallback(_check_directory_html, "-IMM")
4904 d.addCallback(_check_directory_html, "-RO")
4906 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4907 d.addCallback(_check_directory_json, expect_rw_uri=False)
4909 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4910 d.addCallback(_check_json, expect_rw_uri=False)
4912 # TODO: check that getting t=info from the Info link in the ro directory
4913 # works, and does not include the writecap URI.
4916 def test_immutable_unknown(self):
4917 return self.test_unknown(immutable=True)
4919 def test_mutant_dirnodes_are_omitted(self):
4920 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4923 c = self.g.clients[0]
4928 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4929 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4930 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4932 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4933 # test the dirnode and web layers separately.
4935 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4936 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4937 # When the directory is read, the mutants should be silently disposed of, leaving
4938 # their lonely sibling.
4939 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4940 # because immutable directories don't have a writecap and therefore that field
4941 # isn't (and can't be) decrypted.
4942 # TODO: The field still exists in the netstring. Technically we should check what
4943 # happens if something is put there (_unpack_contents should raise ValueError),
4944 # but that can wait.
4946 lonely_child = nm.create_from_cap(lonely_uri)
4947 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4948 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4950 def _by_hook_or_by_crook():
4952 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4953 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4955 mutant_write_in_ro_child.get_write_uri = lambda: None
4956 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4958 kids = {u"lonely": (lonely_child, {}),
4959 u"ro": (mutant_ro_child, {}),
4960 u"write-in-ro": (mutant_write_in_ro_child, {}),
4962 d = c.create_immutable_dirnode(kids)
4965 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4966 self.failIf(dn.is_mutable())
4967 self.failUnless(dn.is_readonly())
4968 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4969 self.failIf(hasattr(dn._node, 'get_writekey'))
4971 self.failUnlessIn("RO-IMM", rep)
4973 self.failUnlessIn("CHK", cap.to_string())
4976 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4977 return download_to_data(dn._node)
4978 d.addCallback(_created)
4980 def _check_data(data):
4981 # Decode the netstring representation of the directory to check that all children
4982 # are present. This is a bit of an abstraction violation, but there's not really
4983 # any other way to do it given that the real DirectoryNode._unpack_contents would
4984 # strip the mutant children out (which is what we're trying to test, later).
4987 while position < len(data):
4988 entries, position = split_netstring(data, 1, position)
4990 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4991 name = name_utf8.decode("utf-8")
4992 self.failUnlessEqual(rwcapdata, "")
4993 self.failUnlessIn(name, kids)
4994 (expected_child, ign) = kids[name]
4995 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4998 self.failUnlessReallyEqual(numkids, 3)
4999 return self.rootnode.list()
5000 d.addCallback(_check_data)
5002 # Now when we use the real directory listing code, the mutants should be absent.
5003 def _check_kids(children):
5004 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5005 lonely_node, lonely_metadata = children[u"lonely"]
5007 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5008 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5009 d.addCallback(_check_kids)
5011 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5012 d.addCallback(lambda n: n.list())
5013 d.addCallback(_check_kids) # again with dirnode recreated from cap
5015 # Make sure the lonely child can be listed in HTML...
5016 d.addCallback(lambda ign: self.GET(self.rooturl))
5017 def _check_html(res):
5018 self.failIfIn("URI:SSK", res)
5019 get_lonely = "".join([r'<td>FILE</td>',
5021 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5023 r'\s+<td align="right">%d</td>' % len("one"),
5025 self.failUnless(re.search(get_lonely, res), res)
5027 # find the More Info link for name, should be relative
5028 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5029 info_url = mo.group(1)
5030 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5031 d.addCallback(_check_html)
5034 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5035 def _check_json(res):
5036 data = simplejson.loads(res)
5037 self.failUnlessEqual(data[0], "dirnode")
5038 listed_children = data[1]["children"]
5039 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5040 ll_type, ll_data = listed_children[u"lonely"]
5041 self.failUnlessEqual(ll_type, "filenode")
5042 self.failIfIn("rw_uri", ll_data)
5043 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5044 d.addCallback(_check_json)
5047 def test_deep_check(self):
5048 self.basedir = "web/Grid/deep_check"
5050 c0 = self.g.clients[0]
5054 d = c0.create_dirnode()
5055 def _stash_root_and_create_file(n):
5057 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5058 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5059 d.addCallback(_stash_root_and_create_file)
5060 def _stash_uri(fn, which):
5061 self.uris[which] = fn.get_uri()
5063 d.addCallback(_stash_uri, "good")
5064 d.addCallback(lambda ign:
5065 self.rootnode.add_file(u"small",
5066 upload.Data("literal",
5068 d.addCallback(_stash_uri, "small")
5069 d.addCallback(lambda ign:
5070 self.rootnode.add_file(u"sick",
5071 upload.Data(DATA+"1",
5073 d.addCallback(_stash_uri, "sick")
5075 # this tests that deep-check and stream-manifest will ignore
5076 # UnknownNode instances. Hopefully this will also cover deep-stats.
5077 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5078 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5080 def _clobber_shares(ignored):
5081 self.delete_shares_numbered(self.uris["sick"], [0,1])
5082 d.addCallback(_clobber_shares)
5090 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5093 units = [simplejson.loads(line)
5094 for line in res.splitlines()
5097 print "response is:", res
5098 print "undecodeable line was '%s'" % line
5100 self.failUnlessReallyEqual(len(units), 5+1)
5101 # should be parent-first
5103 self.failUnlessEqual(u0["path"], [])
5104 self.failUnlessEqual(u0["type"], "directory")
5105 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5106 u0cr = u0["check-results"]
5107 self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5108 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5110 ugood = [u for u in units
5111 if u["type"] == "file" and u["path"] == [u"good"]][0]
5112 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5113 ugoodcr = ugood["check-results"]
5114 self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5115 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5118 self.failUnlessEqual(stats["type"], "stats")
5120 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5121 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5122 self.failUnlessReallyEqual(s["count-directories"], 1)
5123 self.failUnlessReallyEqual(s["count-unknown"], 1)
5124 d.addCallback(_done)
5126 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5127 def _check_manifest(res):
5128 self.failUnless(res.endswith("\n"))
5129 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5130 self.failUnlessReallyEqual(len(units), 5+1)
5131 self.failUnlessEqual(units[-1]["type"], "stats")
5133 self.failUnlessEqual(first["path"], [])
5134 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5135 self.failUnlessEqual(first["type"], "directory")
5136 stats = units[-1]["stats"]
5137 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5138 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5139 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5140 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5141 self.failUnlessReallyEqual(stats["count-unknown"], 1)
5142 d.addCallback(_check_manifest)
5144 # now add root/subdir and root/subdir/grandchild, then make subdir
5145 # unrecoverable, then see what happens
5147 d.addCallback(lambda ign:
5148 self.rootnode.create_subdirectory(u"subdir"))
5149 d.addCallback(_stash_uri, "subdir")
5150 d.addCallback(lambda subdir_node:
5151 subdir_node.add_file(u"grandchild",
5152 upload.Data(DATA+"2",
5154 d.addCallback(_stash_uri, "grandchild")
5156 d.addCallback(lambda ign:
5157 self.delete_shares_numbered(self.uris["subdir"],
5165 # root/subdir [unrecoverable]
5166 # root/subdir/grandchild
5168 # how should a streaming-JSON API indicate fatal error?
5169 # answer: emit ERROR: instead of a JSON string
5171 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5172 def _check_broken_manifest(res):
5173 lines = res.splitlines()
5175 for (i,line) in enumerate(lines)
5176 if line.startswith("ERROR:")]
5178 self.fail("no ERROR: in output: %s" % (res,))
5179 first_error = error_lines[0]
5180 error_line = lines[first_error]
5181 error_msg = lines[first_error+1:]
5182 error_msg_s = "\n".join(error_msg) + "\n"
5183 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5185 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5186 units = [simplejson.loads(line) for line in lines[:first_error]]
5187 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5188 last_unit = units[-1]
5189 self.failUnlessEqual(last_unit["path"], ["subdir"])
5190 d.addCallback(_check_broken_manifest)
5192 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5193 def _check_broken_deepcheck(res):
5194 lines = res.splitlines()
5196 for (i,line) in enumerate(lines)
5197 if line.startswith("ERROR:")]
5199 self.fail("no ERROR: in output: %s" % (res,))
5200 first_error = error_lines[0]
5201 error_line = lines[first_error]
5202 error_msg = lines[first_error+1:]
5203 error_msg_s = "\n".join(error_msg) + "\n"
5204 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5206 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5207 units = [simplejson.loads(line) for line in lines[:first_error]]
5208 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5209 last_unit = units[-1]
5210 self.failUnlessEqual(last_unit["path"], ["subdir"])
5211 r = last_unit["check-results"]["results"]
5212 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5213 self.failUnlessReallyEqual(r["count-happiness"], 1)
5214 self.failUnlessReallyEqual(r["count-shares-good"], 1)
5215 self.failUnlessReallyEqual(r["recoverable"], False)
5216 d.addCallback(_check_broken_deepcheck)
5218 d.addErrback(self.explain_web_error)
5221 def test_deep_check_and_repair(self):
5222 self.basedir = "web/Grid/deep_check_and_repair"
5224 c0 = self.g.clients[0]
5228 d = c0.create_dirnode()
5229 def _stash_root_and_create_file(n):
5231 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5232 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5233 d.addCallback(_stash_root_and_create_file)
5234 def _stash_uri(fn, which):
5235 self.uris[which] = fn.get_uri()
5236 d.addCallback(_stash_uri, "good")
5237 d.addCallback(lambda ign:
5238 self.rootnode.add_file(u"small",
5239 upload.Data("literal",
5241 d.addCallback(_stash_uri, "small")
5242 d.addCallback(lambda ign:
5243 self.rootnode.add_file(u"sick",
5244 upload.Data(DATA+"1",
5246 d.addCallback(_stash_uri, "sick")
5247 #d.addCallback(lambda ign:
5248 # self.rootnode.add_file(u"dead",
5249 # upload.Data(DATA+"2",
5251 #d.addCallback(_stash_uri, "dead")
5253 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5254 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5255 #d.addCallback(_stash_uri, "corrupt")
5257 def _clobber_shares(ignored):
5258 good_shares = self.find_uri_shares(self.uris["good"])
5259 self.failUnlessReallyEqual(len(good_shares), 10)
5260 sick_shares = self.find_uri_shares(self.uris["sick"])
5261 os.unlink(sick_shares[0][2])
5262 #dead_shares = self.find_uri_shares(self.uris["dead"])
5263 #for i in range(1, 10):
5264 # os.unlink(dead_shares[i][2])
5266 #c_shares = self.find_uri_shares(self.uris["corrupt"])
5267 #cso = CorruptShareOptions()
5268 #cso.stdout = StringIO()
5269 #cso.parseOptions([c_shares[0][2]])
5271 d.addCallback(_clobber_shares)
5274 # root/good CHK, 10 shares
5276 # root/sick CHK, 9 shares
5278 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5280 units = [simplejson.loads(line)
5281 for line in res.splitlines()
5283 self.failUnlessReallyEqual(len(units), 4+1)
5284 # should be parent-first
5286 self.failUnlessEqual(u0["path"], [])
5287 self.failUnlessEqual(u0["type"], "directory")
5288 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5289 u0crr = u0["check-and-repair-results"]
5290 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5291 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5292 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5294 ugood = [u for u in units
5295 if u["type"] == "file" and u["path"] == [u"good"]][0]
5296 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5297 ugoodcrr = ugood["check-and-repair-results"]
5298 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5299 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5300 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5302 usick = [u for u in units
5303 if u["type"] == "file" and u["path"] == [u"sick"]][0]
5304 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5305 usickcrr = usick["check-and-repair-results"]
5306 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5307 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5308 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5309 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5310 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5311 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5314 self.failUnlessEqual(stats["type"], "stats")
5316 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5317 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5318 self.failUnlessReallyEqual(s["count-directories"], 1)
5319 d.addCallback(_done)
5321 d.addErrback(self.explain_web_error)
5324 def _count_leases(self, ignored, which):
5325 u = self.uris[which]
5326 shares = self.find_uri_shares(u)
5328 for shnum, serverid, fn in shares:
5329 sf = get_share_file(fn)
5330 num_leases = len(list(sf.get_leases()))
5331 lease_counts.append( (fn, num_leases) )
5334 def _assert_leasecount(self, lease_counts, expected):
5335 for (fn, num_leases) in lease_counts:
5336 if num_leases != expected:
5337 self.fail("expected %d leases, have %d, on %s" %
5338 (expected, num_leases, fn))
5340 def test_add_lease(self):
5341 self.basedir = "web/Grid/add_lease"
5342 self.set_up_grid(num_clients=2)
5343 c0 = self.g.clients[0]
5346 d = c0.upload(upload.Data(DATA, convergence=""))
5347 def _stash_uri(ur, which):
5348 self.uris[which] = ur.get_uri()
5349 d.addCallback(_stash_uri, "one")
5350 d.addCallback(lambda ign:
5351 c0.upload(upload.Data(DATA+"1", convergence="")))
5352 d.addCallback(_stash_uri, "two")
5353 def _stash_mutable_uri(n, which):
5354 self.uris[which] = n.get_uri()
5355 assert isinstance(self.uris[which], str)
5356 d.addCallback(lambda ign:
5357 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5358 d.addCallback(_stash_mutable_uri, "mutable")
5360 def _compute_fileurls(ignored):
5362 for which in self.uris:
5363 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5364 d.addCallback(_compute_fileurls)
5366 d.addCallback(self._count_leases, "one")
5367 d.addCallback(self._assert_leasecount, 1)
5368 d.addCallback(self._count_leases, "two")
5369 d.addCallback(self._assert_leasecount, 1)
5370 d.addCallback(self._count_leases, "mutable")
5371 d.addCallback(self._assert_leasecount, 1)
5373 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5374 def _got_html_good(res):
5375 self.failUnlessIn("Healthy", res)
5376 self.failIfIn("Not Healthy", res)
5377 d.addCallback(_got_html_good)
5379 d.addCallback(self._count_leases, "one")
5380 d.addCallback(self._assert_leasecount, 1)
5381 d.addCallback(self._count_leases, "two")
5382 d.addCallback(self._assert_leasecount, 1)
5383 d.addCallback(self._count_leases, "mutable")
5384 d.addCallback(self._assert_leasecount, 1)
5386 # this CHECK uses the original client, which uses the same
5387 # lease-secrets, so it will just renew the original lease
5388 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5389 d.addCallback(_got_html_good)
5391 d.addCallback(self._count_leases, "one")
5392 d.addCallback(self._assert_leasecount, 1)
5393 d.addCallback(self._count_leases, "two")
5394 d.addCallback(self._assert_leasecount, 1)
5395 d.addCallback(self._count_leases, "mutable")
5396 d.addCallback(self._assert_leasecount, 1)
5398 # this CHECK uses an alternate client, which adds a second lease
5399 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5400 d.addCallback(_got_html_good)
5402 d.addCallback(self._count_leases, "one")
5403 d.addCallback(self._assert_leasecount, 2)
5404 d.addCallback(self._count_leases, "two")
5405 d.addCallback(self._assert_leasecount, 1)
5406 d.addCallback(self._count_leases, "mutable")
5407 d.addCallback(self._assert_leasecount, 1)
5409 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5410 d.addCallback(_got_html_good)
5412 d.addCallback(self._count_leases, "one")
5413 d.addCallback(self._assert_leasecount, 2)
5414 d.addCallback(self._count_leases, "two")
5415 d.addCallback(self._assert_leasecount, 1)
5416 d.addCallback(self._count_leases, "mutable")
5417 d.addCallback(self._assert_leasecount, 1)
5419 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5421 d.addCallback(_got_html_good)
5423 d.addCallback(self._count_leases, "one")
5424 d.addCallback(self._assert_leasecount, 2)
5425 d.addCallback(self._count_leases, "two")
5426 d.addCallback(self._assert_leasecount, 1)
5427 d.addCallback(self._count_leases, "mutable")
5428 d.addCallback(self._assert_leasecount, 2)
5430 d.addErrback(self.explain_web_error)
5433 def test_deep_add_lease(self):
5434 self.basedir = "web/Grid/deep_add_lease"
5435 self.set_up_grid(num_clients=2)
5436 c0 = self.g.clients[0]
5440 d = c0.create_dirnode()
5441 def _stash_root_and_create_file(n):
5443 self.uris["root"] = n.get_uri()
5444 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5445 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5446 d.addCallback(_stash_root_and_create_file)
5447 def _stash_uri(fn, which):
5448 self.uris[which] = fn.get_uri()
5449 d.addCallback(_stash_uri, "one")
5450 d.addCallback(lambda ign:
5451 self.rootnode.add_file(u"small",
5452 upload.Data("literal",
5454 d.addCallback(_stash_uri, "small")
5456 d.addCallback(lambda ign:
5457 c0.create_mutable_file(publish.MutableData("mutable")))
5458 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5459 d.addCallback(_stash_uri, "mutable")
5461 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5463 units = [simplejson.loads(line)
5464 for line in res.splitlines()
5466 # root, one, small, mutable, stats
5467 self.failUnlessReallyEqual(len(units), 4+1)
5468 d.addCallback(_done)
5470 d.addCallback(self._count_leases, "root")
5471 d.addCallback(self._assert_leasecount, 1)
5472 d.addCallback(self._count_leases, "one")
5473 d.addCallback(self._assert_leasecount, 1)
5474 d.addCallback(self._count_leases, "mutable")
5475 d.addCallback(self._assert_leasecount, 1)
5477 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5478 d.addCallback(_done)
5480 d.addCallback(self._count_leases, "root")
5481 d.addCallback(self._assert_leasecount, 1)
5482 d.addCallback(self._count_leases, "one")
5483 d.addCallback(self._assert_leasecount, 1)
5484 d.addCallback(self._count_leases, "mutable")
5485 d.addCallback(self._assert_leasecount, 1)
5487 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5489 d.addCallback(_done)
5491 d.addCallback(self._count_leases, "root")
5492 d.addCallback(self._assert_leasecount, 2)
5493 d.addCallback(self._count_leases, "one")
5494 d.addCallback(self._assert_leasecount, 2)
5495 d.addCallback(self._count_leases, "mutable")
5496 d.addCallback(self._assert_leasecount, 2)
5498 d.addErrback(self.explain_web_error)
5502 def test_exceptions(self):
5503 self.basedir = "web/Grid/exceptions"
5504 self.set_up_grid(num_clients=1, num_servers=2)
5505 c0 = self.g.clients[0]
5506 c0.encoding_params['happy'] = 2
5509 d = c0.create_dirnode()
5511 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5512 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5514 d.addCallback(_stash_root)
5515 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5517 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5518 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5520 u = uri.from_string(ur.get_uri())
5521 u.key = testutil.flip_bit(u.key, 0)
5522 baduri = u.to_string()
5523 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5524 d.addCallback(_stash_bad)
5525 d.addCallback(lambda ign: c0.create_dirnode())
5526 def _mangle_dirnode_1share(n):
5528 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5529 self.fileurls["dir-1share-json"] = url + "?t=json"
5530 self.delete_shares_numbered(u, range(1,10))
5531 d.addCallback(_mangle_dirnode_1share)
5532 d.addCallback(lambda ign: c0.create_dirnode())
5533 def _mangle_dirnode_0share(n):
5535 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5536 self.fileurls["dir-0share-json"] = url + "?t=json"
5537 self.delete_shares_numbered(u, range(0,10))
5538 d.addCallback(_mangle_dirnode_0share)
5540 # NotEnoughSharesError should be reported sensibly, with a
5541 # text/plain explanation of the problem, and perhaps some
5542 # information on which shares *could* be found.
5544 d.addCallback(lambda ignored:
5545 self.shouldHTTPError("GET unrecoverable",
5546 410, "Gone", "NoSharesError",
5547 self.GET, self.fileurls["0shares"]))
5548 def _check_zero_shares(body):
5549 self.failIfIn("<html>", body)
5550 body = " ".join(body.strip().split())
5551 exp = ("NoSharesError: no shares could be found. "
5552 "Zero shares usually indicates a corrupt URI, or that "
5553 "no servers were connected, but it might also indicate "
5554 "severe corruption. You should perform a filecheck on "
5555 "this object to learn more. The full error message is: "
5556 "no shares (need 3). Last failure: None")
5557 self.failUnlessReallyEqual(exp, body)
5558 d.addCallback(_check_zero_shares)
5561 d.addCallback(lambda ignored:
5562 self.shouldHTTPError("GET 1share",
5563 410, "Gone", "NotEnoughSharesError",
5564 self.GET, self.fileurls["1share"]))
5565 def _check_one_share(body):
5566 self.failIfIn("<html>", body)
5567 body = " ".join(body.strip().split())
5568 msgbase = ("NotEnoughSharesError: This indicates that some "
5569 "servers were unavailable, or that shares have been "
5570 "lost to server departure, hard drive failure, or disk "
5571 "corruption. You should perform a filecheck on "
5572 "this object to learn more. The full error message is:"
5574 msg1 = msgbase + (" ran out of shares:"
5577 " overdue= unused= need 3. Last failure: None")
5578 msg2 = msgbase + (" ran out of shares:"
5580 " pending=Share(sh0-on-xgru5)"
5581 " overdue= unused= need 3. Last failure: None")
5582 self.failUnless(body == msg1 or body == msg2, body)
5583 d.addCallback(_check_one_share)
5585 d.addCallback(lambda ignored:
5586 self.shouldHTTPError("GET imaginary",
5587 404, "Not Found", None,
5588 self.GET, self.fileurls["imaginary"]))
5589 def _missing_child(body):
5590 self.failUnlessIn("No such child: imaginary", body)
5591 d.addCallback(_missing_child)
5593 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5594 def _check_0shares_dir_html(body):
5595 self.failUnlessIn(DIR_HTML_TAG, body)
5596 # we should see the regular page, but without the child table or
5598 body = " ".join(body.strip().split())
5599 self.failUnlessIn('href="?t=info">More info on this directory',
5601 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5602 "could not be retrieved, because there were insufficient "
5603 "good shares. This might indicate that no servers were "
5604 "connected, insufficient servers were connected, the URI "
5605 "was corrupt, or that shares have been lost due to server "
5606 "departure, hard drive failure, or disk corruption. You "
5607 "should perform a filecheck on this object to learn more.")
5608 self.failUnlessIn(exp, body)
5609 self.failUnlessIn("No upload forms: directory is unreadable", body)
5610 d.addCallback(_check_0shares_dir_html)
5612 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5613 def _check_1shares_dir_html(body):
5614 # at some point, we'll split UnrecoverableFileError into 0-shares
5615 # and some-shares like we did for immutable files (since there
5616 # are different sorts of advice to offer in each case). For now,
5617 # they present the same way.
5618 self.failUnlessIn(DIR_HTML_TAG, body)
5619 body = " ".join(body.strip().split())
5620 self.failUnlessIn('href="?t=info">More info on this directory',
5622 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5623 "could not be retrieved, because there were insufficient "
5624 "good shares. This might indicate that no servers were "
5625 "connected, insufficient servers were connected, the URI "
5626 "was corrupt, or that shares have been lost due to server "
5627 "departure, hard drive failure, or disk corruption. You "
5628 "should perform a filecheck on this object to learn more.")
5629 self.failUnlessIn(exp, body)
5630 self.failUnlessIn("No upload forms: directory is unreadable", body)
5631 d.addCallback(_check_1shares_dir_html)
5633 d.addCallback(lambda ignored:
5634 self.shouldHTTPError("GET dir-0share-json",
5635 410, "Gone", "UnrecoverableFileError",
5637 self.fileurls["dir-0share-json"]))
5638 def _check_unrecoverable_file(body):
5639 self.failIfIn("<html>", body)
5640 body = " ".join(body.strip().split())
5641 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5642 "could not be retrieved, because there were insufficient "
5643 "good shares. This might indicate that no servers were "
5644 "connected, insufficient servers were connected, the URI "
5645 "was corrupt, or that shares have been lost due to server "
5646 "departure, hard drive failure, or disk corruption. You "
5647 "should perform a filecheck on this object to learn more.")
5648 self.failUnlessReallyEqual(exp, body)
5649 d.addCallback(_check_unrecoverable_file)
5651 d.addCallback(lambda ignored:
5652 self.shouldHTTPError("GET dir-1share-json",
5653 410, "Gone", "UnrecoverableFileError",
5655 self.fileurls["dir-1share-json"]))
5656 d.addCallback(_check_unrecoverable_file)
5658 d.addCallback(lambda ignored:
5659 self.shouldHTTPError("GET imaginary",
5660 404, "Not Found", None,
5661 self.GET, self.fileurls["imaginary"]))
5663 # attach a webapi child that throws a random error, to test how it
5665 w = c0.getServiceNamed("webish")
5666 w.root.putChild("ERRORBOOM", ErrorBoom())
5668 # "Accept: */*" : should get a text/html stack trace
5669 # "Accept: text/plain" : should get a text/plain stack trace
5670 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5671 # no Accept header: should get a text/html stack trace
5673 d.addCallback(lambda ignored:
5674 self.shouldHTTPError("GET errorboom_html",
5675 500, "Internal Server Error", None,
5676 self.GET, "ERRORBOOM",
5677 headers={"accept": "*/*"}))
5678 def _internal_error_html1(body):
5679 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5680 d.addCallback(_internal_error_html1)
5682 d.addCallback(lambda ignored:
5683 self.shouldHTTPError("GET errorboom_text",
5684 500, "Internal Server Error", None,
5685 self.GET, "ERRORBOOM",
5686 headers={"accept": "text/plain"}))
5687 def _internal_error_text2(body):
5688 self.failIfIn("<html>", body)
5689 self.failUnless(body.startswith("Traceback "), body)
5690 d.addCallback(_internal_error_text2)
5692 CLI_accepts = "text/plain, application/octet-stream"
5693 d.addCallback(lambda ignored:
5694 self.shouldHTTPError("GET errorboom_text",
5695 500, "Internal Server Error", None,
5696 self.GET, "ERRORBOOM",
5697 headers={"accept": CLI_accepts}))
5698 def _internal_error_text3(body):
5699 self.failIfIn("<html>", body)
5700 self.failUnless(body.startswith("Traceback "), body)
5701 d.addCallback(_internal_error_text3)
5703 d.addCallback(lambda ignored:
5704 self.shouldHTTPError("GET errorboom_text",
5705 500, "Internal Server Error", None,
5706 self.GET, "ERRORBOOM"))
5707 def _internal_error_html4(body):
5708 self.failUnlessIn("<html>", body)
5709 d.addCallback(_internal_error_html4)
5711 def _flush_errors(res):
5712 # Trial: please ignore the CompletelyUnhandledError in the logs
5713 self.flushLoggedErrors(CompletelyUnhandledError)
5715 d.addBoth(_flush_errors)
5719 def test_blacklist(self):
5720 # download from a blacklisted URI, get an error
5721 self.basedir = "web/Grid/blacklist"
5723 c0 = self.g.clients[0]
5724 c0_basedir = c0.basedir
5725 fn = os.path.join(c0_basedir, "access.blacklist")
5727 DATA = "off-limits " * 50
5729 d = c0.upload(upload.Data(DATA, convergence=""))
5730 def _stash_uri_and_create_dir(ur):
5731 self.uri = ur.get_uri()
5732 self.url = "uri/"+self.uri
5733 u = uri.from_string_filenode(self.uri)
5734 self.si = u.get_storage_index()
5735 childnode = c0.create_node_from_uri(self.uri, None)
5736 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5737 d.addCallback(_stash_uri_and_create_dir)
5738 def _stash_dir(node):
5739 self.dir_node = node
5740 self.dir_uri = node.get_uri()
5741 self.dir_url = "uri/"+self.dir_uri
5742 d.addCallback(_stash_dir)
5743 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5744 def _check_dir_html(body):
5745 self.failUnlessIn(DIR_HTML_TAG, body)
5746 self.failUnlessIn("blacklisted.txt</a>", body)
5747 d.addCallback(_check_dir_html)
5748 d.addCallback(lambda ign: self.GET(self.url))
5749 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5751 def _blacklist(ign):
5753 f.write(" # this is a comment\n")
5755 f.write("\n") # also exercise blank lines
5756 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5758 # clients should be checking the blacklist each time, so we don't
5759 # need to restart the client
5760 d.addCallback(_blacklist)
5761 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5763 "Access Prohibited: off-limits",
5764 self.GET, self.url))
5766 # We should still be able to list the parent directory, in HTML...
5767 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5768 def _check_dir_html2(body):
5769 self.failUnlessIn(DIR_HTML_TAG, body)
5770 self.failUnlessIn("blacklisted.txt</strike>", body)
5771 d.addCallback(_check_dir_html2)
5773 # ... and in JSON (used by CLI).
5774 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5775 def _check_dir_json(res):
5776 data = simplejson.loads(res)
5777 self.failUnless(isinstance(data, list), data)
5778 self.failUnlessEqual(data[0], "dirnode")
5779 self.failUnless(isinstance(data[1], dict), data)
5780 self.failUnlessIn("children", data[1])
5781 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5782 childdata = data[1]["children"]["blacklisted.txt"]
5783 self.failUnless(isinstance(childdata, list), data)
5784 self.failUnlessEqual(childdata[0], "filenode")
5785 self.failUnless(isinstance(childdata[1], dict), data)
5786 d.addCallback(_check_dir_json)
5788 def _unblacklist(ign):
5789 open(fn, "w").close()
5790 # the Blacklist object watches mtime to tell when the file has
5791 # changed, but on windows this test will run faster than the
5792 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5793 # to force a reload.
5794 self.g.clients[0].blacklist.last_mtime -= 2.0
5795 d.addCallback(_unblacklist)
5797 # now a read should work
5798 d.addCallback(lambda ign: self.GET(self.url))
5799 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5801 # read again to exercise the blacklist-is-unchanged logic
5802 d.addCallback(lambda ign: self.GET(self.url))
5803 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5805 # now add a blacklisted directory, and make sure files under it are
5808 childnode = c0.create_node_from_uri(self.uri, None)
5809 return c0.create_dirnode({u"child": (childnode,{}) })
5810 d.addCallback(_add_dir)
5811 def _get_dircap(dn):
5812 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5813 self.dir_url_base = "uri/"+dn.get_write_uri()
5814 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5815 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5816 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5817 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5818 d.addCallback(_get_dircap)
5819 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5820 d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5821 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5822 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5823 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5824 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5825 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5826 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5827 d.addCallback(lambda ign: self.GET(self.child_url))
5828 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5830 def _block_dir(ign):
5832 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5834 self.g.clients[0].blacklist.last_mtime -= 2.0
5835 d.addCallback(_block_dir)
5836 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5838 "Access Prohibited: dir-off-limits",
5839 self.GET, self.dir_url_base))
5840 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5842 "Access Prohibited: dir-off-limits",
5843 self.GET, self.dir_url_json1))
5844 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5846 "Access Prohibited: dir-off-limits",
5847 self.GET, self.dir_url_json2))
5848 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5850 "Access Prohibited: dir-off-limits",
5851 self.GET, self.dir_url_json_ro))
5852 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5854 "Access Prohibited: dir-off-limits",
5855 self.GET, self.child_url))
5859 class CompletelyUnhandledError(Exception):
5861 class ErrorBoom(rend.Page):
5862 def beforeRender(self, ctx):
5863 raise CompletelyUnhandledError("whoops")