1 import os.path, re, urllib, time, cgi
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow.util import escapeToXML
15 from nevow import rend
17 from allmydata import interfaces, uri, webish, dirnode
18 from allmydata.storage.shares import get_share_file
19 from allmydata.storage_client import StorageFarmBroker, StubServer
20 from allmydata.immutable import upload
21 from allmydata.immutable.downloader.status import DownloadStatus
22 from allmydata.dirnode import DirectoryNode
23 from allmydata.nodemaker import NodeMaker
24 from allmydata.unknown import UnknownNode
25 from allmydata.web import status, common
26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
27 from allmydata.util import fileutil, base32, hashutil
28 from allmydata.util.consumer import download_to_data
29 from allmydata.util.netstring import split_netstring
30 from allmydata.util.encodingutil import to_str
31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
32 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
33 make_mutable_file_uri, create_mutable_filenode
34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
35 from allmydata.mutable import servermap, publish, retrieve
36 import allmydata.test.common_util as testutil
37 from allmydata.test.no_network import GridTestMixin
38 from allmydata.test.common_web import HTTPClientGETFactory, \
40 from allmydata.client import Client, SecretHolder
41 from allmydata.introducer import IntroducerNode
43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
44 # create a webserver that works against them
46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
53 DIR_HTML_TAG = '<html lang="en">'
55 class FakeStatsProvider:
57 stats = {'stats': {}, 'counters': {}}
60 class FakeNodeMaker(NodeMaker):
65 'max_segment_size':128*1024 # 1024=KiB
67 def _create_lit(self, cap):
68 return FakeCHKFileNode(cap, self.all_contents)
69 def _create_immutable(self, cap):
70 return FakeCHKFileNode(cap, self.all_contents)
71 def _create_mutable(self, cap):
72 return FakeMutableFileNode(None, None,
73 self.encoding_params, None,
74 self.all_contents).init_from_cap(cap)
75 def create_mutable_file(self, contents="", keysize=None,
76 version=SDMF_VERSION):
77 n = FakeMutableFileNode(None, None, self.encoding_params, None,
79 return n.create(contents, version=version)
81 class FakeUploader(service.Service):
84 helper_connected = False
86 def upload(self, uploadable):
87 d = uploadable.get_size()
88 d.addCallback(lambda size: uploadable.read(size))
91 n = create_chk_filenode(data, self.all_contents)
92 ur = upload.UploadResults(file_size=len(data),
99 uri_extension_data={},
100 uri_extension_hash="fake",
101 verifycapstr="fakevcap")
102 ur.set_uri(n.get_uri())
104 d.addCallback(_got_data)
107 def get_helper_info(self):
108 return (self.helper_furl, self.helper_connected)
112 ds = DownloadStatus("storage_index", 1234)
115 serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
116 serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
117 storage_index = hashutil.storage_index_hash("SI")
118 e0 = ds.add_segment_request(0, now)
120 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
121 e1 = ds.add_segment_request(1, now+2)
123 # two outstanding requests
124 e2 = ds.add_segment_request(2, now+4)
125 e3 = ds.add_segment_request(3, now+5)
126 del e2,e3 # hush pyflakes
128 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
129 e = ds.add_segment_request(4, now)
131 e.deliver(now, 0, 140, 0.5)
133 e = ds.add_dyhb_request(serverA, now)
134 e.finished([1,2], now+1)
135 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
137 e = ds.add_read_event(0, 120, now)
138 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
140 e = ds.add_read_event(120, 30, now+2) # left unfinished
142 e = ds.add_block_request(serverA, 1, 100, 20, now)
143 e.finished(20, now+1)
144 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
146 # make sure that add_read_event() can come first too
147 ds1 = DownloadStatus(storage_index, 1234)
148 e = ds1.add_read_event(0, 120, now)
149 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
155 _all_upload_status = [upload.UploadStatus()]
156 _all_download_status = [build_one_ds()]
157 _all_mapupdate_statuses = [servermap.UpdateStatus()]
158 _all_publish_statuses = [publish.PublishStatus()]
159 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
161 def list_all_upload_statuses(self):
162 return self._all_upload_status
163 def list_all_download_statuses(self):
164 return self._all_download_status
165 def list_all_mapupdate_statuses(self):
166 return self._all_mapupdate_statuses
167 def list_all_publish_statuses(self):
168 return self._all_publish_statuses
169 def list_all_retrieve_statuses(self):
170 return self._all_retrieve_statuses
171 def list_all_helper_statuses(self):
174 class FakeDisplayableServer(StubServer):
175 def __init__(self, serverid, nickname):
176 StubServer.__init__(self, serverid)
177 self.announcement = {"my-version": "allmydata-tahoe-fake",
178 "service-name": "storage",
179 "nickname": nickname}
180 def is_connected(self):
182 def get_permutation_seed(self):
184 def get_remote_host(self):
186 def get_last_loss_time(self):
188 def get_announcement_time(self):
190 def get_announcement(self):
191 return self.announcement
192 def get_nickname(self):
193 return self.announcement["nickname"]
194 def get_available_space(self):
197 class FakeBucketCounter(object):
199 return {"last-complete-bucket-count": 0}
200 def get_progress(self):
201 return {"estimated-time-per-cycle": 0,
202 "cycle-in-progress": False,
203 "remaining-wait-time": 0}
205 class FakeLeaseChecker(object):
207 self.expiration_enabled = False
209 self.override_lease_duration = None
210 self.sharetypes_to_expire = {}
212 return {"history": None}
213 def get_progress(self):
214 return {"estimated-time-per-cycle": 0,
215 "cycle-in-progress": False,
216 "remaining-wait-time": 0}
218 class FakeStorageServer(service.MultiService):
220 def __init__(self, nodeid, nickname):
221 service.MultiService.__init__(self)
222 self.my_nodeid = nodeid
223 self.nickname = nickname
224 self.bucket_counter = FakeBucketCounter()
225 self.lease_checker = FakeLeaseChecker()
227 return {"storage_server.accepting_immutable_shares": False}
229 class FakeClient(Client):
231 # don't upcall to Client.__init__, since we only want to initialize a
233 service.MultiService.__init__(self)
234 self.all_contents = {}
235 self.nodeid = "fake_nodeid"
236 self.nickname = u"fake_nickname \u263A"
237 self.introducer_furl = "None"
238 self.stats_provider = FakeStatsProvider()
239 self._secret_holder = SecretHolder("lease secret", "convergence secret")
241 self.convergence = "some random string"
242 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
243 # fake knowledge of another server
244 self.storage_broker.test_add_server("other_nodeid",
245 FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
246 self.introducer_client = None
247 self.history = FakeHistory()
248 self.uploader = FakeUploader()
249 self.uploader.all_contents = self.all_contents
250 self.uploader.setServiceParent(self)
251 self.blacklist = None
252 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
255 self.nodemaker.all_contents = self.all_contents
256 self.mutable_file_default = SDMF_VERSION
257 self.addService(FakeStorageServer(self.nodeid, self.nickname))
259 def get_long_nodeid(self):
261 def get_long_tubid(self):
264 def startService(self):
265 return service.MultiService.startService(self)
266 def stopService(self):
267 return service.MultiService.stopService(self)
269 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
271 class WebMixin(object):
273 self.s = FakeClient()
274 self.s.startService()
275 self.staticdir = self.mktemp()
277 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
279 self.ws.setServiceParent(self.s)
280 self.webish_port = self.ws.getPortnum()
281 self.webish_url = self.ws.getURL()
282 assert self.webish_url.endswith("/")
283 self.webish_url = self.webish_url[:-1] # these tests add their own /
285 l = [ self.s.create_dirnode() for x in range(6) ]
286 d = defer.DeferredList(l)
288 self.public_root = res[0][1]
289 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
290 self.public_url = "/uri/" + self.public_root.get_uri()
291 self.private_root = res[1][1]
295 self._foo_uri = foo.get_uri()
296 self._foo_readonly_uri = foo.get_readonly_uri()
297 self._foo_verifycap = foo.get_verify_cap().to_string()
298 # NOTE: we ignore the deferred on all set_uri() calls, because we
299 # know the fake nodes do these synchronously
300 self.public_root.set_uri(u"foo", foo.get_uri(),
301 foo.get_readonly_uri())
303 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
304 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
305 self._bar_txt_verifycap = n.get_verify_cap().to_string()
308 # XXX: Do we ever use this?
309 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
311 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
314 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
315 assert self._quux_txt_uri.startswith("URI:MDMF")
316 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
318 foo.set_uri(u"empty", res[3][1].get_uri(),
319 res[3][1].get_readonly_uri())
320 sub_uri = res[4][1].get_uri()
321 self._sub_uri = sub_uri
322 foo.set_uri(u"sub", sub_uri, sub_uri)
323 sub = self.s.create_node_from_uri(sub_uri)
326 _ign, n, blocking_uri = self.makefile(1)
327 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
329 # filenode to test for html encoding issues
330 self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
331 self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
332 self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
333 self._htmlname_escaped = escapeToXML(self._htmlname_raw)
334 self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
335 self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
336 self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
337 foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
339 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
340 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
341 # still think of it as an umlaut
342 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
344 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
345 self._baz_file_uri = baz_file
346 sub.set_uri(u"baz.txt", baz_file, baz_file)
348 _ign, n, self._bad_file_uri = self.makefile(3)
349 # this uri should not be downloadable
350 del self.s.all_contents[self._bad_file_uri]
353 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
354 rodir.get_readonly_uri())
355 rodir.set_uri(u"nor", baz_file, baz_file)
361 # public/foo/quux.txt
362 # public/foo/blockingfile
363 # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
366 # public/foo/sub/baz.txt
368 # public/reedownlee/nor
369 self.NEWFILE_CONTENTS = "newfile contents\n"
371 return foo.get_metadata_for(u"bar.txt")
373 def _got_metadata(metadata):
374 self._bar_txt_metadata = metadata
375 d.addCallback(_got_metadata)
378 def get_all_contents(self):
379 return self.s.all_contents
381 def makefile(self, number):
382 contents = "contents of file %s\n" % number
383 n = create_chk_filenode(contents, self.get_all_contents())
384 return contents, n, n.get_uri()
386 def makefile_mutable(self, number, mdmf=False):
387 contents = "contents of mutable file %s\n" % number
388 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
389 return contents, n, n.get_uri(), n.get_readonly_uri()
392 return self.s.stopService()
394 def failUnlessIsBarDotTxt(self, res):
395 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
397 def failUnlessIsQuuxDotTxt(self, res):
398 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
400 def failUnlessIsBazDotTxt(self, res):
401 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
403 def failUnlessIsSubBazDotTxt(self, res):
404 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
406 def failUnlessIsBarJSON(self, res):
407 data = simplejson.loads(res)
408 self.failUnless(isinstance(data, list))
409 self.failUnlessEqual(data[0], "filenode")
410 self.failUnless(isinstance(data[1], dict))
411 self.failIf(data[1]["mutable"])
412 self.failIfIn("rw_uri", data[1]) # immutable
413 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
414 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
415 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
417 def failUnlessIsQuuxJSON(self, res, readonly=False):
418 data = simplejson.loads(res)
419 self.failUnless(isinstance(data, list))
420 self.failUnlessEqual(data[0], "filenode")
421 self.failUnless(isinstance(data[1], dict))
423 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
425 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
426 self.failUnless(metadata['mutable'])
428 self.failIfIn("rw_uri", metadata)
430 self.failUnlessIn("rw_uri", metadata)
431 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
432 self.failUnlessIn("ro_uri", metadata)
433 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
434 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
436 def failUnlessIsFooJSON(self, res):
437 data = simplejson.loads(res)
438 self.failUnless(isinstance(data, list))
439 self.failUnlessEqual(data[0], "dirnode", res)
440 self.failUnless(isinstance(data[1], dict))
441 self.failUnless(data[1]["mutable"])
442 self.failUnlessIn("rw_uri", data[1]) # mutable
443 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
444 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
445 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
447 kidnames = sorted([unicode(n) for n in data[1]["children"]])
448 self.failUnlessEqual(kidnames,
449 [self._htmlname_unicode, u"bar.txt", u"baz.txt",
450 u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
451 kids = dict( [(unicode(name),value)
453 in data[1]["children"].iteritems()] )
454 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
455 self.failUnlessIn("metadata", kids[u"sub"][1])
456 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
457 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
458 self.failUnlessIn("linkcrtime", tahoe_md)
459 self.failUnlessIn("linkmotime", tahoe_md)
460 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
461 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
462 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
463 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
464 self._bar_txt_verifycap)
465 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
466 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
467 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
468 self._bar_txt_metadata["tahoe"]["linkcrtime"])
469 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
471 self.failUnlessIn("quux.txt", kids)
472 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
474 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
475 self._quux_txt_readonly_uri)
477 def GET(self, urlpath, followRedirect=False, return_response=False,
479 # if return_response=True, this fires with (data, statuscode,
480 # respheaders) instead of just data.
481 assert not isinstance(urlpath, unicode)
482 url = self.webish_url + urlpath
483 factory = HTTPClientGETFactory(url, method="GET",
484 followRedirect=followRedirect, **kwargs)
485 reactor.connectTCP("localhost", self.webish_port, factory)
488 return (data, factory.status, factory.response_headers)
490 d.addCallback(_got_data)
491 return factory.deferred
493 def HEAD(self, urlpath, return_response=False, **kwargs):
494 # this requires some surgery, because twisted.web.client doesn't want
495 # to give us back the response headers.
496 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
497 reactor.connectTCP("localhost", self.webish_port, factory)
500 return (data, factory.status, factory.response_headers)
502 d.addCallback(_got_data)
503 return factory.deferred
505 def PUT(self, urlpath, data, **kwargs):
506 url = self.webish_url + urlpath
507 return client.getPage(url, method="PUT", postdata=data, **kwargs)
509 def DELETE(self, urlpath):
510 url = self.webish_url + urlpath
511 return client.getPage(url, method="DELETE")
513 def POST(self, urlpath, followRedirect=False, **fields):
514 sepbase = "boogabooga"
518 form.append('Content-Disposition: form-data; name="_charset"')
522 for name, value in fields.iteritems():
523 if isinstance(value, tuple):
524 filename, value = value
525 form.append('Content-Disposition: form-data; name="%s"; '
526 'filename="%s"' % (name, filename.encode("utf-8")))
528 form.append('Content-Disposition: form-data; name="%s"' % name)
530 if isinstance(value, unicode):
531 value = value.encode("utf-8")
534 assert isinstance(value, str)
541 body = "\r\n".join(form) + "\r\n"
542 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
543 return self.POST2(urlpath, body, headers, followRedirect)
545 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
546 url = self.webish_url + urlpath
547 return client.getPage(url, method="POST", postdata=body,
548 headers=headers, followRedirect=followRedirect)
550 def shouldFail(self, res, expected_failure, which,
551 substring=None, response_substring=None):
552 if isinstance(res, failure.Failure):
553 res.trap(expected_failure)
555 self.failUnlessIn(substring, str(res), which)
556 if response_substring:
557 self.failUnlessIn(response_substring, res.value.response, which)
559 self.fail("%s was supposed to raise %s, not get '%s'" %
560 (which, expected_failure, res))
562 def shouldFail2(self, expected_failure, which, substring,
564 callable, *args, **kwargs):
565 assert substring is None or isinstance(substring, str)
566 assert response_substring is None or isinstance(response_substring, str)
567 d = defer.maybeDeferred(callable, *args, **kwargs)
569 if isinstance(res, failure.Failure):
570 res.trap(expected_failure)
572 self.failUnlessIn(substring, str(res),
573 "'%s' not in '%s' (response is '%s') for test '%s'" % \
574 (substring, str(res),
575 getattr(res.value, "response", ""),
577 if response_substring:
578 self.failUnlessIn(response_substring, res.value.response,
579 "'%s' not in '%s' for test '%s'" % \
580 (response_substring, res.value.response,
583 self.fail("%s was supposed to raise %s, not get '%s'" %
584 (which, expected_failure, res))
588 def should404(self, res, which):
589 if isinstance(res, failure.Failure):
590 res.trap(error.Error)
591 self.failUnlessReallyEqual(res.value.status, "404")
593 self.fail("%s was supposed to Error(404), not get '%s'" %
596 def should302(self, res, which):
597 if isinstance(res, failure.Failure):
598 res.trap(error.Error)
599 self.failUnlessReallyEqual(res.value.status, "302")
601 self.fail("%s was supposed to Error(302), not get '%s'" %
605 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
606 def test_create(self):
609 def test_welcome(self):
612 self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
613 self.failUnlessIn(FAVICON_MARKUP, res)
614 self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
615 self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
616 self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
617 self.failUnlessIn('Page rendered at', res)
618 self.failUnlessIn('Tahoe-LAFS code imported from:', res)
619 res_u = res.decode('utf-8')
620 self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
621 self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
622 self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
623 self.failUnlessIn('<td><h3>Available</h3></td>', res)
624 self.failUnlessIn('123.5kB', res)
626 self.s.basedir = 'web/test_welcome'
627 fileutil.make_dirs("web/test_welcome")
628 fileutil.make_dirs("web/test_welcome/private")
630 d.addCallback(_check)
633 def test_introducer_status(self):
634 class MockIntroducerClient(object):
635 def __init__(self, connected):
636 self.connected = connected
637 def connected_to_introducer(self):
638 return self.connected
640 d = defer.succeed(None)
642 # introducer not connected, unguessable furl
643 def _set_introducer_not_connected_unguessable(ign):
644 self.s.introducer_furl = "pb://someIntroducer/secret"
645 self.s.introducer_client = MockIntroducerClient(False)
647 d.addCallback(_set_introducer_not_connected_unguessable)
648 def _check_introducer_not_connected_unguessable(res):
649 html = res.replace('\n', ' ')
650 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
651 self.failIfIn('pb://someIntroducer/secret', html)
652 self.failUnless(re.search('<img src="img/connected-no.png" alt="Disconnected" />', html), res)
653 d.addCallback(_check_introducer_not_connected_unguessable)
655 # introducer connected, unguessable furl
656 def _set_introducer_connected_unguessable(ign):
657 self.s.introducer_furl = "pb://someIntroducer/secret"
658 self.s.introducer_client = MockIntroducerClient(True)
660 d.addCallback(_set_introducer_connected_unguessable)
661 def _check_introducer_connected_unguessable(res):
662 html = res.replace('\n', ' ')
663 self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
664 self.failIfIn('pb://someIntroducer/secret', html)
665 self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
666 d.addCallback(_check_introducer_connected_unguessable)
668 # introducer connected, guessable furl
669 def _set_introducer_connected_guessable(ign):
670 self.s.introducer_furl = "pb://someIntroducer/introducer"
671 self.s.introducer_client = MockIntroducerClient(True)
673 d.addCallback(_set_introducer_connected_guessable)
674 def _check_introducer_connected_guessable(res):
675 html = res.replace('\n', ' ')
676 self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
677 self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
678 d.addCallback(_check_introducer_connected_guessable)
681 def test_helper_status(self):
682 d = defer.succeed(None)
684 # set helper furl to None
685 def _set_no_helper(ign):
686 self.s.uploader.helper_furl = None
688 d.addCallback(_set_no_helper)
689 def _check_no_helper(res):
690 html = res.replace('\n', ' ')
691 self.failUnless(re.search('<img src="img/connected-not-configured.png" alt="Not Configured" />', html), res)
692 d.addCallback(_check_no_helper)
694 # enable helper, not connected
695 def _set_helper_not_connected(ign):
696 self.s.uploader.helper_furl = "pb://someHelper/secret"
697 self.s.uploader.helper_connected = False
699 d.addCallback(_set_helper_not_connected)
700 def _check_helper_not_connected(res):
701 html = res.replace('\n', ' ')
702 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
703 self.failIfIn('pb://someHelper/secret', html)
704 self.failUnless(re.search('<img src="img/connected-no.png" alt="Disconnected" />', html), res)
705 d.addCallback(_check_helper_not_connected)
707 # enable helper, connected
708 def _set_helper_connected(ign):
709 self.s.uploader.helper_furl = "pb://someHelper/secret"
710 self.s.uploader.helper_connected = True
712 d.addCallback(_set_helper_connected)
713 def _check_helper_connected(res):
714 html = res.replace('\n', ' ')
715 self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
716 self.failIfIn('pb://someHelper/secret', html)
717 self.failUnless(re.search('<img src="img/connected-yes.png" alt="Connected" />', html), res)
718 d.addCallback(_check_helper_connected)
721 def test_storage(self):
722 d = self.GET("/storage")
724 self.failUnlessIn('Storage Server Status', res)
725 self.failUnlessIn(FAVICON_MARKUP, res)
726 res_u = res.decode('utf-8')
727 self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
728 d.addCallback(_check)
731 def test_status(self):
732 h = self.s.get_history()
733 dl_num = h.list_all_download_statuses()[0].get_counter()
734 ul_num = h.list_all_upload_statuses()[0].get_counter()
735 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
736 pub_num = h.list_all_publish_statuses()[0].get_counter()
737 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
738 d = self.GET("/status", followRedirect=True)
740 self.failUnlessIn('Recent and Active Operations', res)
741 self.failUnlessIn('"down-%d"' % dl_num, res)
742 self.failUnlessIn('"up-%d"' % ul_num, res)
743 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
744 self.failUnlessIn('"publish-%d"' % pub_num, res)
745 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
746 d.addCallback(_check)
747 d.addCallback(lambda res: self.GET("/status/?t=json"))
748 def _check_json(res):
749 data = simplejson.loads(res)
750 self.failUnless(isinstance(data, dict))
751 #active = data["active"]
752 # TODO: test more. We need a way to fake an active operation
754 d.addCallback(_check_json)
756 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
758 self.failUnlessIn("File Download Status", res)
759 d.addCallback(_check_dl)
760 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
761 def _check_dl_json(res):
762 data = simplejson.loads(res)
763 self.failUnless(isinstance(data, dict))
764 self.failUnlessIn("read", data)
765 self.failUnlessEqual(data["read"][0]["length"], 120)
766 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
767 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
768 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
769 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
770 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
771 # serverids[] keys are strings, since that's what JSON does, but
772 # we'd really like them to be ints
773 self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
774 self.failUnless(data["serverids"].has_key("1"),
775 str(data["serverids"]))
776 self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
777 str(data["serverids"]))
778 self.failUnlessEqual(data["server_info"][phwr_id]["short"],
780 self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
782 self.failUnlessIn("dyhb", data)
783 self.failUnlessIn("misc", data)
784 d.addCallback(_check_dl_json)
785 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
787 self.failUnlessIn("File Upload Status", res)
788 d.addCallback(_check_ul)
789 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
790 def _check_mapupdate(res):
791 self.failUnlessIn("Mutable File Servermap Update Status", res)
792 d.addCallback(_check_mapupdate)
793 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
794 def _check_publish(res):
795 self.failUnlessIn("Mutable File Publish Status", res)
796 d.addCallback(_check_publish)
797 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
798 def _check_retrieve(res):
799 self.failUnlessIn("Mutable File Retrieve Status", res)
800 d.addCallback(_check_retrieve)
804 def test_status_numbers(self):
805 drrm = status.DownloadResultsRendererMixin()
806 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
807 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
808 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
809 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
810 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
811 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
812 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
813 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
814 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
816 urrm = status.UploadResultsRendererMixin()
817 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
818 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
819 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
820 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
821 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
822 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
823 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
824 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
825 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
827 def test_GET_FILEURL(self):
828 d = self.GET(self.public_url + "/foo/bar.txt")
829 d.addCallback(self.failUnlessIsBarDotTxt)
832 def test_GET_FILEURL_range(self):
833 headers = {"range": "bytes=1-10"}
834 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
835 return_response=True)
836 def _got((res, status, headers)):
837 self.failUnlessReallyEqual(int(status), 206)
838 self.failUnless(headers.has_key("content-range"))
839 self.failUnlessReallyEqual(headers["content-range"][0],
840 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
841 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
845 def test_GET_FILEURL_partial_range(self):
846 headers = {"range": "bytes=5-"}
847 length = len(self.BAR_CONTENTS)
848 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
849 return_response=True)
850 def _got((res, status, headers)):
851 self.failUnlessReallyEqual(int(status), 206)
852 self.failUnless(headers.has_key("content-range"))
853 self.failUnlessReallyEqual(headers["content-range"][0],
854 "bytes 5-%d/%d" % (length-1, length))
855 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
859 def test_GET_FILEURL_partial_end_range(self):
860 headers = {"range": "bytes=-5"}
861 length = len(self.BAR_CONTENTS)
862 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
863 return_response=True)
864 def _got((res, status, headers)):
865 self.failUnlessReallyEqual(int(status), 206)
866 self.failUnless(headers.has_key("content-range"))
867 self.failUnlessReallyEqual(headers["content-range"][0],
868 "bytes %d-%d/%d" % (length-5, length-1, length))
869 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
873 def test_GET_FILEURL_partial_range_overrun(self):
874 headers = {"range": "bytes=100-200"}
875 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
876 "416 Requested Range not satisfiable",
877 "First beyond end of file",
878 self.GET, self.public_url + "/foo/bar.txt",
882 def test_HEAD_FILEURL_range(self):
883 headers = {"range": "bytes=1-10"}
884 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
885 return_response=True)
886 def _got((res, status, headers)):
887 self.failUnlessReallyEqual(res, "")
888 self.failUnlessReallyEqual(int(status), 206)
889 self.failUnless(headers.has_key("content-range"))
890 self.failUnlessReallyEqual(headers["content-range"][0],
891 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
895 def test_HEAD_FILEURL_partial_range(self):
896 headers = {"range": "bytes=5-"}
897 length = len(self.BAR_CONTENTS)
898 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
899 return_response=True)
900 def _got((res, status, headers)):
901 self.failUnlessReallyEqual(int(status), 206)
902 self.failUnless(headers.has_key("content-range"))
903 self.failUnlessReallyEqual(headers["content-range"][0],
904 "bytes 5-%d/%d" % (length-1, length))
908 def test_HEAD_FILEURL_partial_end_range(self):
909 headers = {"range": "bytes=-5"}
910 length = len(self.BAR_CONTENTS)
911 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
912 return_response=True)
913 def _got((res, status, headers)):
914 self.failUnlessReallyEqual(int(status), 206)
915 self.failUnless(headers.has_key("content-range"))
916 self.failUnlessReallyEqual(headers["content-range"][0],
917 "bytes %d-%d/%d" % (length-5, length-1, length))
921 def test_HEAD_FILEURL_partial_range_overrun(self):
922 headers = {"range": "bytes=100-200"}
923 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
924 "416 Requested Range not satisfiable",
926 self.HEAD, self.public_url + "/foo/bar.txt",
930 def test_GET_FILEURL_range_bad(self):
931 headers = {"range": "BOGUS=fizbop-quarnak"}
932 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
933 return_response=True)
934 def _got((res, status, headers)):
935 self.failUnlessReallyEqual(int(status), 200)
936 self.failUnless(not headers.has_key("content-range"))
937 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
941 def test_HEAD_FILEURL(self):
942 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
943 def _got((res, status, headers)):
944 self.failUnlessReallyEqual(res, "")
945 self.failUnlessReallyEqual(headers["content-length"][0],
946 str(len(self.BAR_CONTENTS)))
947 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
951 def test_GET_FILEURL_named(self):
952 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
953 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
954 d = self.GET(base + "/@@name=/blah.txt")
955 d.addCallback(self.failUnlessIsBarDotTxt)
956 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
957 d.addCallback(self.failUnlessIsBarDotTxt)
958 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
959 d.addCallback(self.failUnlessIsBarDotTxt)
960 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
961 d.addCallback(self.failUnlessIsBarDotTxt)
962 save_url = base + "?save=true&filename=blah.txt"
963 d.addCallback(lambda res: self.GET(save_url))
964 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
965 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
966 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
967 u_url = base + "?save=true&filename=" + u_fn_e
968 d.addCallback(lambda res: self.GET(u_url))
969 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
972 def test_PUT_FILEURL_named_bad(self):
973 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
974 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
976 "/file can only be used with GET or HEAD",
977 self.PUT, base + "/@@name=/blah.txt", "")
981 def test_GET_DIRURL_named_bad(self):
982 base = "/file/%s" % urllib.quote(self._foo_uri)
983 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
986 self.GET, base + "/@@name=/blah.txt")
989 def test_GET_slash_file_bad(self):
990 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
992 "/file must be followed by a file-cap and a name",
996 def test_GET_unhandled_URI_named(self):
997 contents, n, newuri = self.makefile(12)
998 verifier_cap = n.get_verify_cap().to_string()
999 base = "/file/%s" % urllib.quote(verifier_cap)
1000 # client.create_node_from_uri() can't handle verify-caps
1001 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1002 "400 Bad Request", "is not a file-cap",
1006 def test_GET_unhandled_URI(self):
1007 contents, n, newuri = self.makefile(12)
1008 verifier_cap = n.get_verify_cap().to_string()
1009 base = "/uri/%s" % urllib.quote(verifier_cap)
1010 # client.create_node_from_uri() can't handle verify-caps
1011 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1013 "GET unknown URI type: can only do t=info",
1017 def test_GET_FILE_URI(self):
1018 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1020 d.addCallback(self.failUnlessIsBarDotTxt)
1023 def test_GET_FILE_URI_mdmf(self):
1024 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1026 d.addCallback(self.failUnlessIsQuuxDotTxt)
1029 def test_GET_FILE_URI_mdmf_extensions(self):
1030 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1032 d.addCallback(self.failUnlessIsQuuxDotTxt)
1035 def test_GET_FILE_URI_mdmf_readonly(self):
1036 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1038 d.addCallback(self.failUnlessIsQuuxDotTxt)
1041 def test_GET_FILE_URI_badchild(self):
1042 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1043 errmsg = "Files have no children, certainly not named 'boguschild'"
1044 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1045 "400 Bad Request", errmsg,
1049 def test_PUT_FILE_URI_badchild(self):
1050 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1051 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1052 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1053 "400 Bad Request", errmsg,
1057 def test_PUT_FILE_URI_mdmf(self):
1058 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1059 self._quux_new_contents = "new_contents"
1061 d.addCallback(lambda res:
1062 self.failUnlessIsQuuxDotTxt(res))
1063 d.addCallback(lambda ignored:
1064 self.PUT(base, self._quux_new_contents))
1065 d.addCallback(lambda ignored:
1067 d.addCallback(lambda res:
1068 self.failUnlessReallyEqual(res, self._quux_new_contents))
1071 def test_PUT_FILE_URI_mdmf_extensions(self):
1072 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1073 self._quux_new_contents = "new_contents"
1075 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1076 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1077 d.addCallback(lambda ignored: self.GET(base))
1078 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1082 def test_PUT_FILE_URI_mdmf_readonly(self):
1083 # We're not allowed to PUT things to a readonly cap.
1084 base = "/uri/%s" % self._quux_txt_readonly_uri
1086 d.addCallback(lambda res:
1087 self.failUnlessIsQuuxDotTxt(res))
1088 # What should we get here? We get a 500 error now; that's not right.
1089 d.addCallback(lambda ignored:
1090 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1091 "400 Bad Request", "read-only cap",
1092 self.PUT, base, "new data"))
1095 def test_PUT_FILE_URI_sdmf_readonly(self):
1096 # We're not allowed to put things to a readonly cap.
1097 base = "/uri/%s" % self._baz_txt_readonly_uri
1099 d.addCallback(lambda res:
1100 self.failUnlessIsBazDotTxt(res))
1101 d.addCallback(lambda ignored:
1102 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1103 "400 Bad Request", "read-only cap",
1104 self.PUT, base, "new_data"))
1107 def test_GET_etags(self):
1109 def _check_etags(uri):
1111 d2 = _get_etag(uri, 'json')
1112 d = defer.DeferredList([d1, d2], consumeErrors=True)
1113 def _check(results):
1114 # All deferred must succeed
1115 self.failUnless(all([r[0] for r in results]))
1116 # the etag for the t=json form should be just like the etag
1117 # fo the default t='' form, but with a 'json' suffix
1118 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1119 d.addCallback(_check)
1122 def _get_etag(uri, t=''):
1123 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1124 d = self.GET(targetbase, return_response=True, followRedirect=True)
1125 def _just_the_etag(result):
1126 data, response, headers = result
1127 etag = headers['etag'][0]
1128 if uri.startswith('URI:DIR'):
1129 self.failUnless(etag.startswith('DIR:'), etag)
1131 return d.addCallback(_just_the_etag)
1133 # Check that etags work with immutable directories
1134 (newkids, caps) = self._create_immutable_children()
1135 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1136 simplejson.dumps(newkids))
1137 def _stash_immdir_uri(uri):
1138 self._immdir_uri = uri
1140 d.addCallback(_stash_immdir_uri)
1141 d.addCallback(_check_etags)
1143 # Check that etags work with immutable files
1144 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1146 # use the ETag on GET
1147 def _check_match(ign):
1148 uri = "/uri/%s" % self._bar_txt_uri
1149 d = self.GET(uri, return_response=True)
1151 d.addCallback(lambda (data, code, headers):
1153 # do a GET that's supposed to match the ETag
1154 d.addCallback(lambda etag:
1155 self.GET(uri, return_response=True,
1156 headers={"If-None-Match": etag}))
1157 # make sure it short-circuited (304 instead of 200)
1158 d.addCallback(lambda (data, code, headers):
1159 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1161 d.addCallback(_check_match)
1163 def _no_etag(uri, t):
1164 target = "/uri/%s?t=%s" % (uri, t)
1165 d = self.GET(target, return_response=True, followRedirect=True)
1166 d.addCallback(lambda (data, code, headers):
1167 self.failIf("etag" in headers, target))
1169 def _yes_etag(uri, t):
1170 target = "/uri/%s?t=%s" % (uri, t)
1171 d = self.GET(target, return_response=True, followRedirect=True)
1172 d.addCallback(lambda (data, code, headers):
1173 self.failUnless("etag" in headers, target))
1176 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1177 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1178 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1179 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1180 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1182 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1183 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1184 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1185 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1186 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1187 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1191 # TODO: version of this with a Unicode filename
1192 def test_GET_FILEURL_save(self):
1193 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1194 return_response=True)
1195 def _got((res, statuscode, headers)):
1196 content_disposition = headers["content-disposition"][0]
1197 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1198 self.failUnlessIsBarDotTxt(res)
1202 def test_GET_FILEURL_missing(self):
1203 d = self.GET(self.public_url + "/foo/missing")
1204 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1207 def test_GET_FILEURL_info_mdmf(self):
1208 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1210 self.failUnlessIn("mutable file (mdmf)", res)
1211 self.failUnlessIn(self._quux_txt_uri, res)
1212 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1216 def test_GET_FILEURL_info_mdmf_readonly(self):
1217 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1219 self.failUnlessIn("mutable file (mdmf)", res)
1220 self.failIfIn(self._quux_txt_uri, res)
1221 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1225 def test_GET_FILEURL_info_sdmf(self):
1226 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1228 self.failUnlessIn("mutable file (sdmf)", res)
1229 self.failUnlessIn(self._baz_txt_uri, res)
1233 def test_GET_FILEURL_info_mdmf_extensions(self):
1234 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1236 self.failUnlessIn("mutable file (mdmf)", res)
1237 self.failUnlessIn(self._quux_txt_uri, res)
1238 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1242 def test_PUT_overwrite_only_files(self):
1243 # create a directory, put a file in that directory.
1244 contents, n, filecap = self.makefile(8)
1245 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1246 d.addCallback(lambda res:
1247 self.PUT(self.public_url + "/foo/dir/file1.txt",
1248 self.NEWFILE_CONTENTS))
1249 # try to overwrite the file with replace=only-files
1250 # (this should work)
1251 d.addCallback(lambda res:
1252 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1254 d.addCallback(lambda res:
1255 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1256 "There was already a child by that name, and you asked me "
1257 "to not replace it",
1258 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1262 def test_PUT_NEWFILEURL(self):
1263 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1264 # TODO: we lose the response code, so we can't check this
1265 #self.failUnlessReallyEqual(responsecode, 201)
1266 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1267 d.addCallback(lambda res:
1268 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1269 self.NEWFILE_CONTENTS))
1272 def test_PUT_NEWFILEURL_not_mutable(self):
1273 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1274 self.NEWFILE_CONTENTS)
1275 # TODO: we lose the response code, so we can't check this
1276 #self.failUnlessReallyEqual(responsecode, 201)
1277 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1278 d.addCallback(lambda res:
1279 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1280 self.NEWFILE_CONTENTS))
1283 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1284 # this should get us a few segments of an MDMF mutable file,
1285 # which we can then test for.
1286 contents = self.NEWFILE_CONTENTS * 300000
1287 d = self.PUT("/uri?format=mdmf",
1289 def _got_filecap(filecap):
1290 self.failUnless(filecap.startswith("URI:MDMF"))
1292 d.addCallback(_got_filecap)
1293 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1294 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1297 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1298 contents = self.NEWFILE_CONTENTS * 300000
1299 d = self.PUT("/uri?format=sdmf",
1301 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1302 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1305 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1306 contents = self.NEWFILE_CONTENTS * 300000
1307 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1308 400, "Bad Request", "Unknown format: foo",
1309 self.PUT, "/uri?format=foo",
1312 def test_PUT_NEWFILEURL_range_bad(self):
1313 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1314 target = self.public_url + "/foo/new.txt"
1315 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1316 "501 Not Implemented",
1317 "Content-Range in PUT not yet supported",
1318 # (and certainly not for immutable files)
1319 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1321 d.addCallback(lambda res:
1322 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1325 def test_PUT_NEWFILEURL_mutable(self):
1326 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1327 self.NEWFILE_CONTENTS)
1328 # TODO: we lose the response code, so we can't check this
1329 #self.failUnlessReallyEqual(responsecode, 201)
1330 def _check_uri(res):
1331 u = uri.from_string_mutable_filenode(res)
1332 self.failUnless(u.is_mutable())
1333 self.failIf(u.is_readonly())
1335 d.addCallback(_check_uri)
1336 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1337 d.addCallback(lambda res:
1338 self.failUnlessMutableChildContentsAre(self._foo_node,
1340 self.NEWFILE_CONTENTS))
1343 def test_PUT_NEWFILEURL_mutable_toobig(self):
1344 # It is okay to upload large mutable files, so we should be able
1346 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1347 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1350 def test_PUT_NEWFILEURL_replace(self):
1351 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1352 # TODO: we lose the response code, so we can't check this
1353 #self.failUnlessReallyEqual(responsecode, 200)
1354 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1355 d.addCallback(lambda res:
1356 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1357 self.NEWFILE_CONTENTS))
1360 def test_PUT_NEWFILEURL_bad_t(self):
1361 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1362 "PUT to a file: bad t=bogus",
1363 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1367 def test_PUT_NEWFILEURL_no_replace(self):
1368 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1369 self.NEWFILE_CONTENTS)
1370 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1372 "There was already a child by that name, and you asked me "
1373 "to not replace it")
1376 def test_PUT_NEWFILEURL_mkdirs(self):
1377 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1379 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1380 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1381 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1382 d.addCallback(lambda res:
1383 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1384 self.NEWFILE_CONTENTS))
1387 def test_PUT_NEWFILEURL_blocked(self):
1388 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1389 self.NEWFILE_CONTENTS)
1390 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1392 "Unable to create directory 'blockingfile': a file was in the way")
1395 def test_PUT_NEWFILEURL_emptyname(self):
1396 # an empty pathname component (i.e. a double-slash) is disallowed
1397 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1399 "The webapi does not allow empty pathname components",
1400 self.PUT, self.public_url + "/foo//new.txt", "")
1403 def test_DELETE_FILEURL(self):
1404 d = self.DELETE(self.public_url + "/foo/bar.txt")
1405 d.addCallback(lambda res:
1406 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1409 def test_DELETE_FILEURL_missing(self):
1410 d = self.DELETE(self.public_url + "/foo/missing")
1411 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1414 def test_DELETE_FILEURL_missing2(self):
1415 d = self.DELETE(self.public_url + "/missing/missing")
1416 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1419 def failUnlessHasBarDotTxtMetadata(self, res):
1420 data = simplejson.loads(res)
1421 self.failUnless(isinstance(data, list))
1422 self.failUnlessIn("metadata", data[1])
1423 self.failUnlessIn("tahoe", data[1]["metadata"])
1424 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1425 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1426 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1427 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1429 def test_GET_FILEURL_json(self):
1430 # twisted.web.http.parse_qs ignores any query args without an '=', so
1431 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1432 # instead. This may make it tricky to emulate the S3 interface
1434 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1436 self.failUnlessIsBarJSON(data)
1437 self.failUnlessHasBarDotTxtMetadata(data)
1439 d.addCallback(_check1)
1442 def test_GET_FILEURL_json_mutable_type(self):
1443 # The JSON should include format, which says whether the
1444 # file is SDMF or MDMF
1445 d = self.PUT("/uri?format=mdmf",
1446 self.NEWFILE_CONTENTS * 300000)
1447 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1448 def _got_json(json, version):
1449 data = simplejson.loads(json)
1450 assert "filenode" == data[0]
1452 assert isinstance(data, dict)
1454 self.failUnlessIn("format", data)
1455 self.failUnlessEqual(data["format"], version)
1457 d.addCallback(_got_json, "MDMF")
1458 # Now make an SDMF file and check that it is reported correctly.
1459 d.addCallback(lambda ignored:
1460 self.PUT("/uri?format=sdmf",
1461 self.NEWFILE_CONTENTS * 300000))
1462 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1463 d.addCallback(_got_json, "SDMF")
1466 def test_GET_FILEURL_json_mdmf(self):
1467 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1468 d.addCallback(self.failUnlessIsQuuxJSON)
1471 def test_GET_FILEURL_json_missing(self):
1472 d = self.GET(self.public_url + "/foo/missing?json")
1473 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1476 def test_GET_FILEURL_uri(self):
1477 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1479 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1480 d.addCallback(_check)
1481 d.addCallback(lambda res:
1482 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1484 # for now, for files, uris and readonly-uris are the same
1485 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1486 d.addCallback(_check2)
1489 def test_GET_FILEURL_badtype(self):
1490 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1493 self.public_url + "/foo/bar.txt?t=bogus")
1496 def test_CSS_FILE(self):
1497 d = self.GET("/tahoe.css", followRedirect=True)
1499 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1500 self.failUnless(CSS_STYLE.search(res), res)
1501 d.addCallback(_check)
1504 def test_GET_FILEURL_uri_missing(self):
1505 d = self.GET(self.public_url + "/foo/missing?t=uri")
1506 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1509 def _check_upload_and_mkdir_forms(self, html):
1510 # We should have a form to create a file, with radio buttons that allow
1511 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1512 self.failUnlessIn('name="t" value="upload"', html)
1513 self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1514 self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1515 self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1517 # We should also have the ability to create a mutable directory, with
1518 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1519 # or MDMF directory.
1520 self.failUnlessIn('name="t" value="mkdir"', html)
1521 self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1522 self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1524 self.failUnlessIn(FAVICON_MARKUP, html)
1526 def test_GET_DIRECTORY_html(self):
1527 d = self.GET(self.public_url + "/foo", followRedirect=True)
1529 self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1530 self._check_upload_and_mkdir_forms(html)
1531 self.failUnlessIn("quux", html)
1532 d.addCallback(_check)
1535 def test_GET_DIRECTORY_html_filenode_encoding(self):
1536 d = self.GET(self.public_url + "/foo", followRedirect=True)
1538 # Check if encoded entries are there
1539 self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1540 + self._htmlname_escaped + '</a>', html)
1541 self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1542 self.failIfIn(self._htmlname_escaped_double, html)
1543 # Make sure that Nevow escaping actually works by checking for unsafe characters
1544 # and that '&' is escaped.
1546 self.failUnlessIn(entity, self._htmlname_raw)
1547 self.failIfIn(entity, self._htmlname_escaped)
1548 self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1549 self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1550 d.addCallback(_check)
1553 def test_GET_root_html(self):
1555 d.addCallback(self._check_upload_and_mkdir_forms)
1558 def test_GET_DIRURL(self):
1559 # the addSlash means we get a redirect here
1560 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1562 d = self.GET(self.public_url + "/foo", followRedirect=True)
1564 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1566 # the FILE reference points to a URI, but it should end in bar.txt
1567 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1568 (ROOT, urllib.quote(self._bar_txt_uri)))
1569 get_bar = "".join([r'<td>FILE</td>',
1571 r'<a href="%s">bar.txt</a>' % bar_url,
1573 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1575 self.failUnless(re.search(get_bar, res), res)
1576 for label in ['unlink', 'rename/relink']:
1577 for line in res.split("\n"):
1578 # find the line that contains the relevant button for bar.txt
1579 if ("form action" in line and
1580 ('value="%s"' % (label,)) in line and
1581 'value="bar.txt"' in line):
1582 # the form target should use a relative URL
1583 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1584 self.failUnlessIn('action="%s"' % foo_url, line)
1585 # and the when_done= should too
1586 #done_url = urllib.quote(???)
1587 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1589 # 'unlink' needs to use POST because it directly has a side effect
1590 if label == 'unlink':
1591 self.failUnlessIn('method="post"', line)
1594 self.fail("unable to find '%s bar.txt' line" % (label,))
1596 # the DIR reference just points to a URI
1597 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1598 get_sub = ((r'<td>DIR</td>')
1599 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1600 self.failUnless(re.search(get_sub, res), res)
1601 d.addCallback(_check)
1603 # look at a readonly directory
1604 d.addCallback(lambda res:
1605 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1607 self.failUnlessIn("(read-only)", res)
1608 self.failIfIn("Upload a file", res)
1609 d.addCallback(_check2)
1611 # and at a directory that contains a readonly directory
1612 d.addCallback(lambda res:
1613 self.GET(self.public_url, followRedirect=True))
1615 self.failUnless(re.search('<td>DIR-RO</td>'
1616 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1617 d.addCallback(_check3)
1619 # and an empty directory
1620 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1622 self.failUnlessIn("directory is empty", res)
1623 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" class="btn" value="Create" />', re.I)
1624 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1625 d.addCallback(_check4)
1627 # and at a literal directory
1628 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1629 d.addCallback(lambda res:
1630 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1632 self.failUnlessIn('(immutable)', res)
1633 self.failUnless(re.search('<td>FILE</td>'
1634 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1635 d.addCallback(_check5)
1638 def test_GET_DIRURL_badtype(self):
1639 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1643 self.public_url + "/foo?t=bogus")
1646 def test_GET_DIRURL_json(self):
1647 d = self.GET(self.public_url + "/foo?t=json")
1648 d.addCallback(self.failUnlessIsFooJSON)
1651 def test_GET_DIRURL_json_format(self):
1652 d = self.PUT(self.public_url + \
1653 "/foo/sdmf.txt?format=sdmf",
1654 self.NEWFILE_CONTENTS * 300000)
1655 d.addCallback(lambda ignored:
1656 self.PUT(self.public_url + \
1657 "/foo/mdmf.txt?format=mdmf",
1658 self.NEWFILE_CONTENTS * 300000))
1659 # Now we have an MDMF and SDMF file in the directory. If we GET
1660 # its JSON, we should see their encodings.
1661 d.addCallback(lambda ignored:
1662 self.GET(self.public_url + "/foo?t=json"))
1663 def _got_json(json):
1664 data = simplejson.loads(json)
1665 assert data[0] == "dirnode"
1668 kids = data['children']
1670 mdmf_data = kids['mdmf.txt'][1]
1671 self.failUnlessIn("format", mdmf_data)
1672 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1674 sdmf_data = kids['sdmf.txt'][1]
1675 self.failUnlessIn("format", sdmf_data)
1676 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1677 d.addCallback(_got_json)
1681 def test_POST_DIRURL_manifest_no_ophandle(self):
1682 d = self.shouldFail2(error.Error,
1683 "test_POST_DIRURL_manifest_no_ophandle",
1685 "slow operation requires ophandle=",
1686 self.POST, self.public_url, t="start-manifest")
1689 def test_POST_DIRURL_manifest(self):
1690 d = defer.succeed(None)
1691 def getman(ignored, output):
1692 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1693 followRedirect=True)
1694 d.addCallback(self.wait_for_operation, "125")
1695 d.addCallback(self.get_operation_results, "125", output)
1697 d.addCallback(getman, None)
1698 def _got_html(manifest):
1699 self.failUnlessIn("Manifest of SI=", manifest)
1700 self.failUnlessIn("<td>sub</td>", manifest)
1701 self.failUnlessIn(self._sub_uri, manifest)
1702 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1703 self.failUnlessIn(FAVICON_MARKUP, manifest)
1704 d.addCallback(_got_html)
1706 # both t=status and unadorned GET should be identical
1707 d.addCallback(lambda res: self.GET("/operations/125"))
1708 d.addCallback(_got_html)
1710 d.addCallback(getman, "html")
1711 d.addCallback(_got_html)
1712 d.addCallback(getman, "text")
1713 def _got_text(manifest):
1714 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1715 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1716 d.addCallback(_got_text)
1717 d.addCallback(getman, "JSON")
1719 data = res["manifest"]
1721 for (path_list, cap) in data:
1722 got[tuple(path_list)] = cap
1723 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1724 self.failUnlessIn((u"sub", u"baz.txt"), got)
1725 self.failUnlessIn("finished", res)
1726 self.failUnlessIn("origin", res)
1727 self.failUnlessIn("storage-index", res)
1728 self.failUnlessIn("verifycaps", res)
1729 self.failUnlessIn("stats", res)
1730 d.addCallback(_got_json)
1733 def test_POST_DIRURL_deepsize_no_ophandle(self):
1734 d = self.shouldFail2(error.Error,
1735 "test_POST_DIRURL_deepsize_no_ophandle",
1737 "slow operation requires ophandle=",
1738 self.POST, self.public_url, t="start-deep-size")
1741 def test_POST_DIRURL_deepsize(self):
1742 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1743 followRedirect=True)
1744 d.addCallback(self.wait_for_operation, "126")
1745 d.addCallback(self.get_operation_results, "126", "json")
1746 def _got_json(data):
1747 self.failUnlessReallyEqual(data["finished"], True)
1749 self.failUnless(size > 1000)
1750 d.addCallback(_got_json)
1751 d.addCallback(self.get_operation_results, "126", "text")
1753 mo = re.search(r'^size: (\d+)$', res, re.M)
1754 self.failUnless(mo, res)
1755 size = int(mo.group(1))
1756 # with directories, the size varies.
1757 self.failUnless(size > 1000)
1758 d.addCallback(_got_text)
1761 def test_POST_DIRURL_deepstats_no_ophandle(self):
1762 d = self.shouldFail2(error.Error,
1763 "test_POST_DIRURL_deepstats_no_ophandle",
1765 "slow operation requires ophandle=",
1766 self.POST, self.public_url, t="start-deep-stats")
1769 def test_POST_DIRURL_deepstats(self):
1770 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1771 followRedirect=True)
1772 d.addCallback(self.wait_for_operation, "127")
1773 d.addCallback(self.get_operation_results, "127", "json")
1774 def _got_json(stats):
1775 expected = {"count-immutable-files": 4,
1776 "count-mutable-files": 2,
1777 "count-literal-files": 0,
1779 "count-directories": 3,
1780 "size-immutable-files": 76,
1781 "size-literal-files": 0,
1782 #"size-directories": 1912, # varies
1783 #"largest-directory": 1590,
1784 "largest-directory-children": 8,
1785 "largest-immutable-file": 19,
1787 for k,v in expected.iteritems():
1788 self.failUnlessReallyEqual(stats[k], v,
1789 "stats[%s] was %s, not %s" %
1791 self.failUnlessReallyEqual(stats["size-files-histogram"],
1793 d.addCallback(_got_json)
1796 def test_POST_DIRURL_stream_manifest(self):
1797 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1799 self.failUnless(res.endswith("\n"))
1800 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1801 self.failUnlessReallyEqual(len(units), 10)
1802 self.failUnlessEqual(units[-1]["type"], "stats")
1804 self.failUnlessEqual(first["path"], [])
1805 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1806 self.failUnlessEqual(first["type"], "directory")
1807 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1808 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1809 self.failIfEqual(baz["storage-index"], None)
1810 self.failIfEqual(baz["verifycap"], None)
1811 self.failIfEqual(baz["repaircap"], None)
1812 # XXX: Add quux and baz to this test.
1814 d.addCallback(_check)
1817 def test_GET_DIRURL_uri(self):
1818 d = self.GET(self.public_url + "/foo?t=uri")
1820 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1821 d.addCallback(_check)
1824 def test_GET_DIRURL_readonly_uri(self):
1825 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1827 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1828 d.addCallback(_check)
1831 def test_PUT_NEWDIRURL(self):
1832 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1833 d.addCallback(lambda res:
1834 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1835 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1836 d.addCallback(self.failUnlessNodeKeysAre, [])
1839 def test_PUT_NEWDIRURL_mdmf(self):
1840 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1841 d.addCallback(lambda res:
1842 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1843 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1844 d.addCallback(lambda node:
1845 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1848 def test_PUT_NEWDIRURL_sdmf(self):
1849 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1851 d.addCallback(lambda res:
1852 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1853 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1854 d.addCallback(lambda node:
1855 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1858 def test_PUT_NEWDIRURL_bad_format(self):
1859 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1860 400, "Bad Request", "Unknown format: foo",
1861 self.PUT, self.public_url +
1862 "/foo/newdir=?t=mkdir&format=foo", "")
1864 def test_POST_NEWDIRURL(self):
1865 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1866 d.addCallback(lambda res:
1867 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1868 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1869 d.addCallback(self.failUnlessNodeKeysAre, [])
1872 def test_POST_NEWDIRURL_mdmf(self):
1873 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1874 d.addCallback(lambda res:
1875 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1876 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1877 d.addCallback(lambda node:
1878 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1881 def test_POST_NEWDIRURL_sdmf(self):
1882 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1883 d.addCallback(lambda res:
1884 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1885 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1886 d.addCallback(lambda node:
1887 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1890 def test_POST_NEWDIRURL_bad_format(self):
1891 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1892 400, "Bad Request", "Unknown format: foo",
1893 self.POST2, self.public_url + \
1894 "/foo/newdir?t=mkdir&format=foo", "")
1896 def test_POST_NEWDIRURL_emptyname(self):
1897 # an empty pathname component (i.e. a double-slash) is disallowed
1898 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1900 "The webapi does not allow empty pathname components, i.e. a double slash",
1901 self.POST, self.public_url + "//?t=mkdir")
1904 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1905 (newkids, caps) = self._create_initial_children()
1906 query = "/foo/newdir?t=mkdir-with-children"
1907 if version == MDMF_VERSION:
1908 query += "&format=mdmf"
1909 elif version == SDMF_VERSION:
1910 query += "&format=sdmf"
1912 version = SDMF_VERSION # for later
1913 d = self.POST2(self.public_url + query,
1914 simplejson.dumps(newkids))
1916 n = self.s.create_node_from_uri(uri.strip())
1917 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1918 self.failUnlessEqual(n._node.get_version(), version)
1919 d2.addCallback(lambda ign:
1920 self.failUnlessROChildURIIs(n, u"child-imm",
1922 d2.addCallback(lambda ign:
1923 self.failUnlessRWChildURIIs(n, u"child-mutable",
1925 d2.addCallback(lambda ign:
1926 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1928 d2.addCallback(lambda ign:
1929 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1930 caps['unknown_rocap']))
1931 d2.addCallback(lambda ign:
1932 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1933 caps['unknown_rwcap']))
1934 d2.addCallback(lambda ign:
1935 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1936 caps['unknown_immcap']))
1937 d2.addCallback(lambda ign:
1938 self.failUnlessRWChildURIIs(n, u"dirchild",
1940 d2.addCallback(lambda ign:
1941 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1943 d2.addCallback(lambda ign:
1944 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1945 caps['emptydircap']))
1947 d.addCallback(_check)
1948 d.addCallback(lambda res:
1949 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1950 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1951 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1952 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1953 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1956 def test_POST_NEWDIRURL_initial_children(self):
1957 return self._do_POST_NEWDIRURL_initial_children_test()
1959 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1960 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1962 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1963 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1965 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1966 (newkids, caps) = self._create_initial_children()
1967 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1968 400, "Bad Request", "Unknown format: foo",
1969 self.POST2, self.public_url + \
1970 "/foo/newdir?t=mkdir-with-children&format=foo",
1971 simplejson.dumps(newkids))
1973 def test_POST_NEWDIRURL_immutable(self):
1974 (newkids, caps) = self._create_immutable_children()
1975 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1976 simplejson.dumps(newkids))
1978 n = self.s.create_node_from_uri(uri.strip())
1979 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1980 d2.addCallback(lambda ign:
1981 self.failUnlessROChildURIIs(n, u"child-imm",
1983 d2.addCallback(lambda ign:
1984 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1985 caps['unknown_immcap']))
1986 d2.addCallback(lambda ign:
1987 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1989 d2.addCallback(lambda ign:
1990 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1992 d2.addCallback(lambda ign:
1993 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1994 caps['emptydircap']))
1996 d.addCallback(_check)
1997 d.addCallback(lambda res:
1998 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2005 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2006 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2007 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2008 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2009 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2010 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2011 d.addErrback(self.explain_web_error)
2014 def test_POST_NEWDIRURL_immutable_bad(self):
2015 (newkids, caps) = self._create_initial_children()
2016 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2018 "needed to be immutable but was not",
2020 self.public_url + "/foo/newdir?t=mkdir-immutable",
2021 simplejson.dumps(newkids))
2024 def test_PUT_NEWDIRURL_exists(self):
2025 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2026 d.addCallback(lambda res:
2027 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2028 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2029 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2032 def test_PUT_NEWDIRURL_blocked(self):
2033 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2034 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2036 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2037 d.addCallback(lambda res:
2038 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2039 d.addCallback(lambda res: self._foo_node.get(u"sub"))
2040 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2043 def test_PUT_NEWDIRURL_mkdirs(self):
2044 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2045 d.addCallback(lambda res:
2046 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2047 d.addCallback(lambda res:
2048 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2049 d.addCallback(lambda res:
2050 self._foo_node.get_child_at_path(u"subdir/newdir"))
2051 d.addCallback(self.failUnlessNodeKeysAre, [])
2054 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2055 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2056 d.addCallback(lambda ignored:
2057 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2058 d.addCallback(lambda ignored:
2059 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2060 d.addCallback(lambda ignored:
2061 self._foo_node.get_child_at_path(u"subdir"))
2062 def _got_subdir(subdir):
2063 # XXX: What we want?
2064 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2065 self.failUnlessNodeHasChild(subdir, u"newdir")
2066 return subdir.get_child_at_path(u"newdir")
2067 d.addCallback(_got_subdir)
2068 d.addCallback(lambda newdir:
2069 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2072 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2073 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2074 d.addCallback(lambda ignored:
2075 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2076 d.addCallback(lambda ignored:
2077 self.failIfNodeHasChild(self._foo_node, u"newdir"))
2078 d.addCallback(lambda ignored:
2079 self._foo_node.get_child_at_path(u"subdir"))
2080 def _got_subdir(subdir):
2081 # XXX: What we want?
2082 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2083 self.failUnlessNodeHasChild(subdir, u"newdir")
2084 return subdir.get_child_at_path(u"newdir")
2085 d.addCallback(_got_subdir)
2086 d.addCallback(lambda newdir:
2087 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2090 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2091 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2092 400, "Bad Request", "Unknown format: foo",
2093 self.PUT, self.public_url + \
2094 "/foo/subdir/newdir?t=mkdir&format=foo",
2097 def test_DELETE_DIRURL(self):
2098 d = self.DELETE(self.public_url + "/foo")
2099 d.addCallback(lambda res:
2100 self.failIfNodeHasChild(self.public_root, u"foo"))
2103 def test_DELETE_DIRURL_missing(self):
2104 d = self.DELETE(self.public_url + "/foo/missing")
2105 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2106 d.addCallback(lambda res:
2107 self.failUnlessNodeHasChild(self.public_root, u"foo"))
2110 def test_DELETE_DIRURL_missing2(self):
2111 d = self.DELETE(self.public_url + "/missing")
2112 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2115 def dump_root(self):
2117 w = webish.DirnodeWalkerMixin()
2118 def visitor(childpath, childnode, metadata):
2120 d = w.walk(self.public_root, visitor)
2123 def failUnlessNodeKeysAre(self, node, expected_keys):
2124 for k in expected_keys:
2125 assert isinstance(k, unicode)
2127 def _check(children):
2128 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2129 d.addCallback(_check)
2131 def failUnlessNodeHasChild(self, node, name):
2132 assert isinstance(name, unicode)
2134 def _check(children):
2135 self.failUnlessIn(name, children)
2136 d.addCallback(_check)
2138 def failIfNodeHasChild(self, node, name):
2139 assert isinstance(name, unicode)
2141 def _check(children):
2142 self.failIfIn(name, children)
2143 d.addCallback(_check)
2146 def failUnlessChildContentsAre(self, node, name, expected_contents):
2147 assert isinstance(name, unicode)
2148 d = node.get_child_at_path(name)
2149 d.addCallback(lambda node: download_to_data(node))
2150 def _check(contents):
2151 self.failUnlessReallyEqual(contents, expected_contents)
2152 d.addCallback(_check)
2155 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2156 assert isinstance(name, unicode)
2157 d = node.get_child_at_path(name)
2158 d.addCallback(lambda node: node.download_best_version())
2159 def _check(contents):
2160 self.failUnlessReallyEqual(contents, expected_contents)
2161 d.addCallback(_check)
2164 def failUnlessRWChildURIIs(self, node, name, expected_uri):
2165 assert isinstance(name, unicode)
2166 d = node.get_child_at_path(name)
2168 self.failUnless(child.is_unknown() or not child.is_readonly())
2169 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2170 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2171 expected_ro_uri = self._make_readonly(expected_uri)
2173 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2174 d.addCallback(_check)
2177 def failUnlessROChildURIIs(self, node, name, expected_uri):
2178 assert isinstance(name, unicode)
2179 d = node.get_child_at_path(name)
2181 self.failUnless(child.is_unknown() or child.is_readonly())
2182 self.failUnlessReallyEqual(child.get_write_uri(), None)
2183 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2184 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2185 d.addCallback(_check)
2188 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2189 assert isinstance(name, unicode)
2190 d = node.get_child_at_path(name)
2192 self.failUnless(child.is_unknown() or not child.is_readonly())
2193 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2194 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2195 expected_ro_uri = self._make_readonly(got_uri)
2197 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2198 d.addCallback(_check)
2201 def failUnlessURIMatchesROChild(self, got_uri, node, name):
2202 assert isinstance(name, unicode)
2203 d = node.get_child_at_path(name)
2205 self.failUnless(child.is_unknown() or child.is_readonly())
2206 self.failUnlessReallyEqual(child.get_write_uri(), None)
2207 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2208 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2209 d.addCallback(_check)
2212 def failUnlessCHKURIHasContents(self, got_uri, contents):
2213 self.failUnless(self.get_all_contents()[got_uri] == contents)
2215 def test_POST_upload(self):
2216 d = self.POST(self.public_url + "/foo", t="upload",
2217 file=("new.txt", self.NEWFILE_CONTENTS))
2219 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2220 d.addCallback(lambda res:
2221 self.failUnlessChildContentsAre(fn, u"new.txt",
2222 self.NEWFILE_CONTENTS))
2225 def test_POST_upload_unicode(self):
2226 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2227 d = self.POST(self.public_url + "/foo", t="upload",
2228 file=(filename, self.NEWFILE_CONTENTS))
2230 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2231 d.addCallback(lambda res:
2232 self.failUnlessChildContentsAre(fn, filename,
2233 self.NEWFILE_CONTENTS))
2234 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2235 d.addCallback(lambda res: self.GET(target_url))
2236 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2237 self.NEWFILE_CONTENTS,
2241 def test_POST_upload_unicode_named(self):
2242 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2243 d = self.POST(self.public_url + "/foo", t="upload",
2245 file=("overridden", self.NEWFILE_CONTENTS))
2247 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2248 d.addCallback(lambda res:
2249 self.failUnlessChildContentsAre(fn, filename,
2250 self.NEWFILE_CONTENTS))
2251 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2252 d.addCallback(lambda res: self.GET(target_url))
2253 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2254 self.NEWFILE_CONTENTS,
2258 def test_POST_upload_no_link(self):
2259 d = self.POST("/uri", t="upload",
2260 file=("new.txt", self.NEWFILE_CONTENTS))
2261 def _check_upload_results(page):
2262 # this should be a page which describes the results of the upload
2263 # that just finished.
2264 self.failUnlessIn("Upload Results:", page)
2265 self.failUnlessIn("URI:", page)
2266 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2267 mo = uri_re.search(page)
2268 self.failUnless(mo, page)
2269 new_uri = mo.group(1)
2271 d.addCallback(_check_upload_results)
2272 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2275 def test_POST_upload_no_link_whendone(self):
2276 d = self.POST("/uri", t="upload", when_done="/",
2277 file=("new.txt", self.NEWFILE_CONTENTS))
2278 d.addBoth(self.shouldRedirect, "/")
2281 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2282 d = defer.maybeDeferred(callable, *args, **kwargs)
2284 if isinstance(res, failure.Failure):
2285 res.trap(error.PageRedirect)
2286 statuscode = res.value.status
2287 target = res.value.location
2288 return checker(statuscode, target)
2289 self.fail("%s: callable was supposed to redirect, not return '%s'"
2294 def test_POST_upload_no_link_whendone_results(self):
2295 def check(statuscode, target):
2296 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2297 self.failUnless(target.startswith(self.webish_url), target)
2298 return client.getPage(target, method="GET")
2299 # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2300 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2302 self.POST, "/uri", t="upload",
2303 when_done="/%75ri/%(uri)s",
2304 file=("new.txt", self.NEWFILE_CONTENTS))
2305 d.addCallback(lambda res:
2306 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2309 def test_POST_upload_no_link_mutable(self):
2310 d = self.POST("/uri", t="upload", mutable="true",
2311 file=("new.txt", self.NEWFILE_CONTENTS))
2312 def _check(filecap):
2313 filecap = filecap.strip()
2314 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2315 self.filecap = filecap
2316 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2317 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2318 n = self.s.create_node_from_uri(filecap)
2319 return n.download_best_version()
2320 d.addCallback(_check)
2322 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2323 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2324 d.addCallback(_check2)
2326 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2327 return self.GET("/file/%s" % urllib.quote(self.filecap))
2328 d.addCallback(_check3)
2330 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2331 d.addCallback(_check4)
2334 def test_POST_upload_no_link_mutable_toobig(self):
2335 # The SDMF size limit is no longer in place, so we should be
2336 # able to upload mutable files that are as large as we want them
2338 d = self.POST("/uri", t="upload", mutable="true",
2339 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2343 def test_POST_upload_format_unlinked(self):
2344 def _check_upload_unlinked(ign, format, uri_prefix):
2345 filename = format + ".txt"
2346 d = self.POST("/uri?t=upload&format=" + format,
2347 file=(filename, self.NEWFILE_CONTENTS * 300000))
2348 def _got_results(results):
2349 if format.upper() in ("SDMF", "MDMF"):
2350 # webapi.rst says this returns a filecap
2353 # for immutable, it returns an "upload results page", and
2354 # the filecap is buried inside
2355 line = [l for l in results.split("\n") if "URI: " in l][0]
2356 mo = re.search(r'<span>([^<]+)</span>', line)
2357 filecap = mo.group(1)
2358 self.failUnless(filecap.startswith(uri_prefix),
2359 (uri_prefix, filecap))
2360 return self.GET("/uri/%s?t=json" % filecap)
2361 d.addCallback(_got_results)
2362 def _got_json(json):
2363 data = simplejson.loads(json)
2365 self.failUnlessIn("format", data)
2366 self.failUnlessEqual(data["format"], format.upper())
2367 d.addCallback(_got_json)
2369 d = defer.succeed(None)
2370 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2371 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2372 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2373 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2376 def test_POST_upload_bad_format_unlinked(self):
2377 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2378 400, "Bad Request", "Unknown format: foo",
2380 "/uri?t=upload&format=foo",
2381 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2383 def test_POST_upload_format(self):
2384 def _check_upload(ign, format, uri_prefix, fn=None):
2385 filename = format + ".txt"
2386 d = self.POST(self.public_url +
2387 "/foo?t=upload&format=" + format,
2388 file=(filename, self.NEWFILE_CONTENTS * 300000))
2389 def _got_filecap(filecap):
2391 filenameu = unicode(filename)
2392 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2393 self.failUnless(filecap.startswith(uri_prefix))
2394 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2395 d.addCallback(_got_filecap)
2396 def _got_json(json):
2397 data = simplejson.loads(json)
2399 self.failUnlessIn("format", data)
2400 self.failUnlessEqual(data["format"], format.upper())
2401 d.addCallback(_got_json)
2404 d = defer.succeed(None)
2405 d.addCallback(_check_upload, "chk", "URI:CHK")
2406 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2407 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2408 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2411 def test_POST_upload_bad_format(self):
2412 return self.shouldHTTPError("POST_upload_bad_format",
2413 400, "Bad Request", "Unknown format: foo",
2414 self.POST, self.public_url + \
2415 "/foo?t=upload&format=foo",
2416 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2418 def test_POST_upload_mutable(self):
2419 # this creates a mutable file
2420 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2421 file=("new.txt", self.NEWFILE_CONTENTS))
2423 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2424 d.addCallback(lambda res:
2425 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2426 self.NEWFILE_CONTENTS))
2427 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2429 self.failUnless(IMutableFileNode.providedBy(newnode))
2430 self.failUnless(newnode.is_mutable())
2431 self.failIf(newnode.is_readonly())
2432 self._mutable_node = newnode
2433 self._mutable_uri = newnode.get_uri()
2436 # now upload it again and make sure that the URI doesn't change
2437 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2438 d.addCallback(lambda res:
2439 self.POST(self.public_url + "/foo", t="upload",
2441 file=("new.txt", NEWER_CONTENTS)))
2442 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2443 d.addCallback(lambda res:
2444 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2446 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2448 self.failUnless(IMutableFileNode.providedBy(newnode))
2449 self.failUnless(newnode.is_mutable())
2450 self.failIf(newnode.is_readonly())
2451 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2452 d.addCallback(_got2)
2454 # upload a second time, using PUT instead of POST
2455 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2456 d.addCallback(lambda res:
2457 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2458 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2459 d.addCallback(lambda res:
2460 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2463 # finally list the directory, since mutable files are displayed
2464 # slightly differently
2466 d.addCallback(lambda res:
2467 self.GET(self.public_url + "/foo/",
2468 followRedirect=True))
2469 def _check_page(res):
2470 # TODO: assert more about the contents
2471 self.failUnlessIn("SSK", res)
2473 d.addCallback(_check_page)
2475 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2477 self.failUnless(IMutableFileNode.providedBy(newnode))
2478 self.failUnless(newnode.is_mutable())
2479 self.failIf(newnode.is_readonly())
2480 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2481 d.addCallback(_got3)
2483 # look at the JSON form of the enclosing directory
2484 d.addCallback(lambda res:
2485 self.GET(self.public_url + "/foo/?t=json",
2486 followRedirect=True))
2487 def _check_page_json(res):
2488 parsed = simplejson.loads(res)
2489 self.failUnlessEqual(parsed[0], "dirnode")
2490 children = dict( [(unicode(name),value)
2492 in parsed[1]["children"].iteritems()] )
2493 self.failUnlessIn(u"new.txt", children)
2494 new_json = children[u"new.txt"]
2495 self.failUnlessEqual(new_json[0], "filenode")
2496 self.failUnless(new_json[1]["mutable"])
2497 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2498 ro_uri = self._mutable_node.get_readonly().to_string()
2499 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2500 d.addCallback(_check_page_json)
2502 # and the JSON form of the file
2503 d.addCallback(lambda res:
2504 self.GET(self.public_url + "/foo/new.txt?t=json"))
2505 def _check_file_json(res):
2506 parsed = simplejson.loads(res)
2507 self.failUnlessEqual(parsed[0], "filenode")
2508 self.failUnless(parsed[1]["mutable"])
2509 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2510 ro_uri = self._mutable_node.get_readonly().to_string()
2511 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2512 d.addCallback(_check_file_json)
2514 # and look at t=uri and t=readonly-uri
2515 d.addCallback(lambda res:
2516 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2517 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2518 d.addCallback(lambda res:
2519 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2520 def _check_ro_uri(res):
2521 ro_uri = self._mutable_node.get_readonly().to_string()
2522 self.failUnlessReallyEqual(res, ro_uri)
2523 d.addCallback(_check_ro_uri)
2525 # make sure we can get to it from /uri/URI
2526 d.addCallback(lambda res:
2527 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2528 d.addCallback(lambda res:
2529 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2531 # and that HEAD computes the size correctly
2532 d.addCallback(lambda res:
2533 self.HEAD(self.public_url + "/foo/new.txt",
2534 return_response=True))
2535 def _got_headers((res, status, headers)):
2536 self.failUnlessReallyEqual(res, "")
2537 self.failUnlessReallyEqual(headers["content-length"][0],
2538 str(len(NEW2_CONTENTS)))
2539 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2540 d.addCallback(_got_headers)
2542 # make sure that outdated size limits aren't enforced anymore.
2543 d.addCallback(lambda ignored:
2544 self.POST(self.public_url + "/foo", t="upload",
2547 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2548 d.addErrback(self.dump_error)
2551 def test_POST_upload_mutable_toobig(self):
2552 # SDMF had a size limti that was removed a while ago. MDMF has
2553 # never had a size limit. Test to make sure that we do not
2554 # encounter errors when trying to upload large mutable files,
2555 # since there should be no coded prohibitions regarding large
2557 d = self.POST(self.public_url + "/foo",
2558 t="upload", mutable="true",
2559 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2562 def dump_error(self, f):
2563 # if the web server returns an error code (like 400 Bad Request),
2564 # web.client.getPage puts the HTTP response body into the .response
2565 # attribute of the exception object that it gives back. It does not
2566 # appear in the Failure's repr(), so the ERROR that trial displays
2567 # will be rather terse and unhelpful. addErrback this method to the
2568 # end of your chain to get more information out of these errors.
2569 if f.check(error.Error):
2570 print "web.error.Error:"
2572 print f.value.response
2575 def test_POST_upload_replace(self):
2576 d = self.POST(self.public_url + "/foo", t="upload",
2577 file=("bar.txt", self.NEWFILE_CONTENTS))
2579 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2580 d.addCallback(lambda res:
2581 self.failUnlessChildContentsAre(fn, u"bar.txt",
2582 self.NEWFILE_CONTENTS))
2585 def test_POST_upload_no_replace_ok(self):
2586 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2587 file=("new.txt", self.NEWFILE_CONTENTS))
2588 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2589 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2590 self.NEWFILE_CONTENTS))
2593 def test_POST_upload_no_replace_queryarg(self):
2594 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2595 file=("bar.txt", self.NEWFILE_CONTENTS))
2596 d.addBoth(self.shouldFail, error.Error,
2597 "POST_upload_no_replace_queryarg",
2599 "There was already a child by that name, and you asked me "
2600 "to not replace it")
2601 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2602 d.addCallback(self.failUnlessIsBarDotTxt)
2605 def test_POST_upload_no_replace_field(self):
2606 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2607 file=("bar.txt", self.NEWFILE_CONTENTS))
2608 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2610 "There was already a child by that name, and you asked me "
2611 "to not replace it")
2612 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2613 d.addCallback(self.failUnlessIsBarDotTxt)
2616 def test_POST_upload_whendone(self):
2617 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2618 file=("new.txt", self.NEWFILE_CONTENTS))
2619 d.addBoth(self.shouldRedirect, "/THERE")
2621 d.addCallback(lambda res:
2622 self.failUnlessChildContentsAre(fn, u"new.txt",
2623 self.NEWFILE_CONTENTS))
2626 def test_POST_upload_named(self):
2628 d = self.POST(self.public_url + "/foo", t="upload",
2629 name="new.txt", file=self.NEWFILE_CONTENTS)
2630 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2631 d.addCallback(lambda res:
2632 self.failUnlessChildContentsAre(fn, u"new.txt",
2633 self.NEWFILE_CONTENTS))
2636 def test_POST_upload_named_badfilename(self):
2637 d = self.POST(self.public_url + "/foo", t="upload",
2638 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2639 d.addBoth(self.shouldFail, error.Error,
2640 "test_POST_upload_named_badfilename",
2642 "name= may not contain a slash",
2644 # make sure that nothing was added
2645 d.addCallback(lambda res:
2646 self.failUnlessNodeKeysAre(self._foo_node,
2647 [self._htmlname_unicode,
2648 u"bar.txt", u"baz.txt", u"blockingfile",
2649 u"empty", u"n\u00fc.txt", u"quux.txt",
2653 def test_POST_FILEURL_check(self):
2654 bar_url = self.public_url + "/foo/bar.txt"
2655 d = self.POST(bar_url, t="check")
2657 self.failUnlessIn("Healthy :", res)
2658 d.addCallback(_check)
2659 redir_url = "http://allmydata.org/TARGET"
2660 def _check2(statuscode, target):
2661 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2662 self.failUnlessReallyEqual(target, redir_url)
2663 d.addCallback(lambda res:
2664 self.shouldRedirect2("test_POST_FILEURL_check",
2668 when_done=redir_url))
2669 d.addCallback(lambda res:
2670 self.POST(bar_url, t="check", return_to=redir_url))
2672 self.failUnlessIn("Healthy :", res)
2673 self.failUnlessIn("Return to file", res)
2674 self.failUnlessIn(redir_url, res)
2675 d.addCallback(_check3)
2677 d.addCallback(lambda res:
2678 self.POST(bar_url, t="check", output="JSON"))
2679 def _check_json(res):
2680 data = simplejson.loads(res)
2681 self.failUnlessIn("storage-index", data)
2682 self.failUnless(data["results"]["healthy"])
2683 d.addCallback(_check_json)
2687 def test_POST_FILEURL_check_and_repair(self):
2688 bar_url = self.public_url + "/foo/bar.txt"
2689 d = self.POST(bar_url, t="check", repair="true")
2691 self.failUnlessIn("Healthy :", res)
2692 d.addCallback(_check)
2693 redir_url = "http://allmydata.org/TARGET"
2694 def _check2(statuscode, target):
2695 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2696 self.failUnlessReallyEqual(target, redir_url)
2697 d.addCallback(lambda res:
2698 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2701 t="check", repair="true",
2702 when_done=redir_url))
2703 d.addCallback(lambda res:
2704 self.POST(bar_url, t="check", return_to=redir_url))
2706 self.failUnlessIn("Healthy :", res)
2707 self.failUnlessIn("Return to file", res)
2708 self.failUnlessIn(redir_url, res)
2709 d.addCallback(_check3)
2712 def test_POST_DIRURL_check(self):
2713 foo_url = self.public_url + "/foo/"
2714 d = self.POST(foo_url, t="check")
2716 self.failUnlessIn("Healthy :", res)
2717 d.addCallback(_check)
2718 redir_url = "http://allmydata.org/TARGET"
2719 def _check2(statuscode, target):
2720 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2721 self.failUnlessReallyEqual(target, redir_url)
2722 d.addCallback(lambda res:
2723 self.shouldRedirect2("test_POST_DIRURL_check",
2727 when_done=redir_url))
2728 d.addCallback(lambda res:
2729 self.POST(foo_url, t="check", return_to=redir_url))
2731 self.failUnlessIn("Healthy :", res)
2732 self.failUnlessIn("Return to file/directory", res)
2733 self.failUnlessIn(redir_url, res)
2734 d.addCallback(_check3)
2736 d.addCallback(lambda res:
2737 self.POST(foo_url, t="check", output="JSON"))
2738 def _check_json(res):
2739 data = simplejson.loads(res)
2740 self.failUnlessIn("storage-index", data)
2741 self.failUnless(data["results"]["healthy"])
2742 d.addCallback(_check_json)
2746 def test_POST_DIRURL_check_and_repair(self):
2747 foo_url = self.public_url + "/foo/"
2748 d = self.POST(foo_url, t="check", repair="true")
2750 self.failUnlessIn("Healthy :", res)
2751 d.addCallback(_check)
2752 redir_url = "http://allmydata.org/TARGET"
2753 def _check2(statuscode, target):
2754 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2755 self.failUnlessReallyEqual(target, redir_url)
2756 d.addCallback(lambda res:
2757 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2760 t="check", repair="true",
2761 when_done=redir_url))
2762 d.addCallback(lambda res:
2763 self.POST(foo_url, t="check", return_to=redir_url))
2765 self.failUnlessIn("Healthy :", res)
2766 self.failUnlessIn("Return to file/directory", res)
2767 self.failUnlessIn(redir_url, res)
2768 d.addCallback(_check3)
2771 def test_POST_FILEURL_mdmf_check(self):
2772 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2773 d = self.POST(quux_url, t="check")
2775 self.failUnlessIn("Healthy", res)
2776 d.addCallback(_check)
2777 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2778 d.addCallback(lambda ignored:
2779 self.POST(quux_extension_url, t="check"))
2780 d.addCallback(_check)
2783 def test_POST_FILEURL_mdmf_check_and_repair(self):
2784 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2785 d = self.POST(quux_url, t="check", repair="true")
2787 self.failUnlessIn("Healthy", res)
2788 d.addCallback(_check)
2789 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2790 d.addCallback(lambda ignored:
2791 self.POST(quux_extension_url, t="check", repair="true"))
2792 d.addCallback(_check)
2795 def wait_for_operation(self, ignored, ophandle):
2796 url = "/operations/" + ophandle
2797 url += "?t=status&output=JSON"
2800 data = simplejson.loads(res)
2801 if not data["finished"]:
2802 d = self.stall(delay=1.0)
2803 d.addCallback(self.wait_for_operation, ophandle)
2809 def get_operation_results(self, ignored, ophandle, output=None):
2810 url = "/operations/" + ophandle
2813 url += "&output=" + output
2816 if output and output.lower() == "json":
2817 return simplejson.loads(res)
2822 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2823 d = self.shouldFail2(error.Error,
2824 "test_POST_DIRURL_deepcheck_no_ophandle",
2826 "slow operation requires ophandle=",
2827 self.POST, self.public_url, t="start-deep-check")
2830 def test_POST_DIRURL_deepcheck(self):
2831 def _check_redirect(statuscode, target):
2832 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2833 self.failUnless(target.endswith("/operations/123"))
2834 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2835 self.POST, self.public_url,
2836 t="start-deep-check", ophandle="123")
2837 d.addCallback(self.wait_for_operation, "123")
2838 def _check_json(data):
2839 self.failUnlessReallyEqual(data["finished"], True)
2840 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2841 self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2842 d.addCallback(_check_json)
2843 d.addCallback(self.get_operation_results, "123", "html")
2844 def _check_html(res):
2845 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2846 self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2847 self.failUnlessIn(FAVICON_MARKUP, res)
2848 d.addCallback(_check_html)
2850 d.addCallback(lambda res:
2851 self.GET("/operations/123/"))
2852 d.addCallback(_check_html) # should be the same as without the slash
2854 d.addCallback(lambda res:
2855 self.shouldFail2(error.Error, "one", "404 Not Found",
2856 "No detailed results for SI bogus",
2857 self.GET, "/operations/123/bogus"))
2859 foo_si = self._foo_node.get_storage_index()
2860 foo_si_s = base32.b2a(foo_si)
2861 d.addCallback(lambda res:
2862 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2863 def _check_foo_json(res):
2864 data = simplejson.loads(res)
2865 self.failUnlessEqual(data["storage-index"], foo_si_s)
2866 self.failUnless(data["results"]["healthy"])
2867 d.addCallback(_check_foo_json)
2870 def test_POST_DIRURL_deepcheck_and_repair(self):
2871 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2872 ophandle="124", output="json", followRedirect=True)
2873 d.addCallback(self.wait_for_operation, "124")
2874 def _check_json(data):
2875 self.failUnlessReallyEqual(data["finished"], True)
2876 self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2877 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2878 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2879 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2880 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2881 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2882 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2883 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2884 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2885 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2886 d.addCallback(_check_json)
2887 d.addCallback(self.get_operation_results, "124", "html")
2888 def _check_html(res):
2889 self.failUnlessIn("Objects Checked: <span>11</span>", res)
2891 self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2892 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2893 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2895 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2896 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2897 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2899 self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2900 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2901 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2903 self.failUnlessIn(FAVICON_MARKUP, res)
2904 d.addCallback(_check_html)
2907 def test_POST_FILEURL_bad_t(self):
2908 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2909 "POST to file: bad t=bogus",
2910 self.POST, self.public_url + "/foo/bar.txt",
2914 def test_POST_mkdir(self): # return value?
2915 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2916 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917 d.addCallback(self.failUnlessNodeKeysAre, [])
2920 def test_POST_mkdir_mdmf(self):
2921 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2922 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2923 d.addCallback(lambda node:
2924 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2927 def test_POST_mkdir_sdmf(self):
2928 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2929 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2930 d.addCallback(lambda node:
2931 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2934 def test_POST_mkdir_bad_format(self):
2935 return self.shouldHTTPError("POST_mkdir_bad_format",
2936 400, "Bad Request", "Unknown format: foo",
2937 self.POST, self.public_url +
2938 "/foo?t=mkdir&name=newdir&format=foo")
2940 def test_POST_mkdir_initial_children(self):
2941 (newkids, caps) = self._create_initial_children()
2942 d = self.POST2(self.public_url +
2943 "/foo?t=mkdir-with-children&name=newdir",
2944 simplejson.dumps(newkids))
2945 d.addCallback(lambda res:
2946 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2947 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2949 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2950 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2953 def test_POST_mkdir_initial_children_mdmf(self):
2954 (newkids, caps) = self._create_initial_children()
2955 d = self.POST2(self.public_url +
2956 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2957 simplejson.dumps(newkids))
2958 d.addCallback(lambda res:
2959 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2960 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2961 d.addCallback(lambda node:
2962 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2963 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2969 def test_POST_mkdir_initial_children_sdmf(self):
2970 (newkids, caps) = self._create_initial_children()
2971 d = self.POST2(self.public_url +
2972 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2973 simplejson.dumps(newkids))
2974 d.addCallback(lambda res:
2975 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2976 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2977 d.addCallback(lambda node:
2978 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2979 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2980 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2984 def test_POST_mkdir_initial_children_bad_format(self):
2985 (newkids, caps) = self._create_initial_children()
2986 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2987 400, "Bad Request", "Unknown format: foo",
2988 self.POST, self.public_url + \
2989 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2990 simplejson.dumps(newkids))
2992 def test_POST_mkdir_immutable(self):
2993 (newkids, caps) = self._create_immutable_children()
2994 d = self.POST2(self.public_url +
2995 "/foo?t=mkdir-immutable&name=newdir",
2996 simplejson.dumps(newkids))
2997 d.addCallback(lambda res:
2998 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2999 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
3001 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3003 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3005 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3006 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3007 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3008 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3009 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3010 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3013 def test_POST_mkdir_immutable_bad(self):
3014 (newkids, caps) = self._create_initial_children()
3015 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3017 "needed to be immutable but was not",
3020 "/foo?t=mkdir-immutable&name=newdir",
3021 simplejson.dumps(newkids))
3024 def test_POST_mkdir_2(self):
3025 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3026 d.addCallback(lambda res:
3027 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3028 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3029 d.addCallback(self.failUnlessNodeKeysAre, [])
3032 def test_POST_mkdirs_2(self):
3033 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3034 d.addCallback(lambda res:
3035 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3036 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3037 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3038 d.addCallback(self.failUnlessNodeKeysAre, [])
3041 def test_POST_mkdir_no_parentdir_noredirect(self):
3042 d = self.POST("/uri?t=mkdir")
3043 def _after_mkdir(res):
3044 uri.DirectoryURI.init_from_string(res)
3045 d.addCallback(_after_mkdir)
3048 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3049 d = self.POST("/uri?t=mkdir&format=mdmf")
3050 def _after_mkdir(res):
3051 u = uri.from_string(res)
3052 # Check that this is an MDMF writecap
3053 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3054 d.addCallback(_after_mkdir)
3057 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3058 d = self.POST("/uri?t=mkdir&format=sdmf")
3059 def _after_mkdir(res):
3060 u = uri.from_string(res)
3061 self.failUnlessIsInstance(u, uri.DirectoryURI)
3062 d.addCallback(_after_mkdir)
3065 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3066 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3067 400, "Bad Request", "Unknown format: foo",
3068 self.POST, self.public_url +
3069 "/uri?t=mkdir&format=foo")
3071 def test_POST_mkdir_no_parentdir_noredirect2(self):
3072 # make sure form-based arguments (as on the welcome page) still work
3073 d = self.POST("/uri", t="mkdir")
3074 def _after_mkdir(res):
3075 uri.DirectoryURI.init_from_string(res)
3076 d.addCallback(_after_mkdir)
3077 d.addErrback(self.explain_web_error)
3080 def test_POST_mkdir_no_parentdir_redirect(self):
3081 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3082 d.addBoth(self.shouldRedirect, None, statuscode='303')
3083 def _check_target(target):
3084 target = urllib.unquote(target)
3085 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3086 d.addCallback(_check_target)
3089 def test_POST_mkdir_no_parentdir_redirect2(self):
3090 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3091 d.addBoth(self.shouldRedirect, None, statuscode='303')
3092 def _check_target(target):
3093 target = urllib.unquote(target)
3094 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3095 d.addCallback(_check_target)
3096 d.addErrback(self.explain_web_error)
3099 def _make_readonly(self, u):
3100 ro_uri = uri.from_string(u).get_readonly()
3103 return ro_uri.to_string()
3105 def _create_initial_children(self):
3106 contents, n, filecap1 = self.makefile(12)
3107 md1 = {"metakey1": "metavalue1"}
3108 filecap2 = make_mutable_file_uri()
3109 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3110 filecap3 = node3.get_readonly_uri()
3111 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3112 dircap = DirectoryNode(node4, None, None).get_uri()
3113 mdmfcap = make_mutable_file_uri(mdmf=True)
3114 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3115 emptydircap = "URI:DIR2-LIT:"
3116 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
3117 "ro_uri": self._make_readonly(filecap1),
3118 "metadata": md1, }],
3119 u"child-mutable": ["filenode", {"rw_uri": filecap2,
3120 "ro_uri": self._make_readonly(filecap2)}],
3121 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3122 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
3123 "ro_uri": unknown_rocap}],
3124 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
3125 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3126 u"dirchild": ["dirnode", {"rw_uri": dircap,
3127 "ro_uri": self._make_readonly(dircap)}],
3128 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3129 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3130 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3131 "ro_uri": self._make_readonly(mdmfcap)}],
3133 return newkids, {'filecap1': filecap1,
3134 'filecap2': filecap2,
3135 'filecap3': filecap3,
3136 'unknown_rwcap': unknown_rwcap,
3137 'unknown_rocap': unknown_rocap,
3138 'unknown_immcap': unknown_immcap,
3140 'litdircap': litdircap,
3141 'emptydircap': emptydircap,
3144 def _create_immutable_children(self):
3145 contents, n, filecap1 = self.makefile(12)
3146 md1 = {"metakey1": "metavalue1"}
3147 tnode = create_chk_filenode("immutable directory contents\n"*10,
3148 self.get_all_contents())
3149 dnode = DirectoryNode(tnode, None, None)
3150 assert not dnode.is_mutable()
3151 immdircap = dnode.get_uri()
3152 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3153 emptydircap = "URI:DIR2-LIT:"
3154 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
3155 "metadata": md1, }],
3156 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
3157 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
3158 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
3159 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
3161 return newkids, {'filecap1': filecap1,
3162 'unknown_immcap': unknown_immcap,
3163 'immdircap': immdircap,
3164 'litdircap': litdircap,
3165 'emptydircap': emptydircap}
3167 def test_POST_mkdir_no_parentdir_initial_children(self):
3168 (newkids, caps) = self._create_initial_children()
3169 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3170 def _after_mkdir(res):
3171 self.failUnless(res.startswith("URI:DIR"), res)
3172 n = self.s.create_node_from_uri(res)
3173 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3174 d2.addCallback(lambda ign:
3175 self.failUnlessROChildURIIs(n, u"child-imm",
3177 d2.addCallback(lambda ign:
3178 self.failUnlessRWChildURIIs(n, u"child-mutable",
3180 d2.addCallback(lambda ign:
3181 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3183 d2.addCallback(lambda ign:
3184 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3185 caps['unknown_rwcap']))
3186 d2.addCallback(lambda ign:
3187 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3188 caps['unknown_rocap']))
3189 d2.addCallback(lambda ign:
3190 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3191 caps['unknown_immcap']))
3192 d2.addCallback(lambda ign:
3193 self.failUnlessRWChildURIIs(n, u"dirchild",
3196 d.addCallback(_after_mkdir)
3199 def test_POST_mkdir_no_parentdir_unexpected_children(self):
3200 # the regular /uri?t=mkdir operation is specified to ignore its body.
3201 # Only t=mkdir-with-children pays attention to it.
3202 (newkids, caps) = self._create_initial_children()
3203 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3205 "t=mkdir does not accept children=, "
3206 "try t=mkdir-with-children instead",
3207 self.POST2, "/uri?t=mkdir", # without children
3208 simplejson.dumps(newkids))
3211 def test_POST_noparent_bad(self):
3212 d = self.shouldHTTPError("POST_noparent_bad",
3214 "/uri accepts only PUT, PUT?t=mkdir, "
3215 "POST?t=upload, and POST?t=mkdir",
3216 self.POST, "/uri?t=bogus")
3219 def test_POST_mkdir_no_parentdir_immutable(self):
3220 (newkids, caps) = self._create_immutable_children()
3221 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3222 def _after_mkdir(res):
3223 self.failUnless(res.startswith("URI:DIR"), res)
3224 n = self.s.create_node_from_uri(res)
3225 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3226 d2.addCallback(lambda ign:
3227 self.failUnlessROChildURIIs(n, u"child-imm",
3229 d2.addCallback(lambda ign:
3230 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3231 caps['unknown_immcap']))
3232 d2.addCallback(lambda ign:
3233 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3235 d2.addCallback(lambda ign:
3236 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3238 d2.addCallback(lambda ign:
3239 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3240 caps['emptydircap']))
3242 d.addCallback(_after_mkdir)
3245 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3246 (newkids, caps) = self._create_initial_children()
3247 d = self.shouldFail2(error.Error,
3248 "test_POST_mkdir_no_parentdir_immutable_bad",
3250 "needed to be immutable but was not",
3252 "/uri?t=mkdir-immutable",
3253 simplejson.dumps(newkids))
3256 def test_welcome_page_mkdir_button(self):
3257 # Fetch the welcome page.
3259 def _after_get_welcome_page(res):
3260 MKDIR_BUTTON_RE = re.compile(
3261 '<form action="([^"]*)" method="post".*'
3262 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3263 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3264 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3265 html = res.replace('\n', ' ')
3266 mo = MKDIR_BUTTON_RE.search(html)
3267 self.failUnless(mo, html)
3268 formaction = mo.group(1)
3270 formaname = mo.group(3)
3271 formavalue = mo.group(4)
3272 return (formaction, formt, formaname, formavalue)
3273 d.addCallback(_after_get_welcome_page)
3274 def _after_parse_form(res):
3275 (formaction, formt, formaname, formavalue) = res
3276 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3277 d.addCallback(_after_parse_form)
3278 d.addBoth(self.shouldRedirect, None, statuscode='303')
3281 def test_POST_mkdir_replace(self): # return value?
3282 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3283 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3284 d.addCallback(self.failUnlessNodeKeysAre, [])
3287 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3288 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3289 d.addBoth(self.shouldFail, error.Error,
3290 "POST_mkdir_no_replace_queryarg",
3292 "There was already a child by that name, and you asked me "
3293 "to not replace it")
3294 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3295 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3298 def test_POST_mkdir_no_replace_field(self): # return value?
3299 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3301 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3303 "There was already a child by that name, and you asked me "
3304 "to not replace it")
3305 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3306 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3309 def test_POST_mkdir_whendone_field(self):
3310 d = self.POST(self.public_url + "/foo",
3311 t="mkdir", name="newdir", when_done="/THERE")
3312 d.addBoth(self.shouldRedirect, "/THERE")
3313 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3314 d.addCallback(self.failUnlessNodeKeysAre, [])
3317 def test_POST_mkdir_whendone_queryarg(self):
3318 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3319 t="mkdir", name="newdir")
3320 d.addBoth(self.shouldRedirect, "/THERE")
3321 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3322 d.addCallback(self.failUnlessNodeKeysAre, [])
3325 def test_POST_bad_t(self):
3326 d = self.shouldFail2(error.Error, "POST_bad_t",
3328 "POST to a directory with bad t=BOGUS",
3329 self.POST, self.public_url + "/foo", t="BOGUS")
3332 def test_POST_set_children(self, command_name="set_children"):
3333 contents9, n9, newuri9 = self.makefile(9)
3334 contents10, n10, newuri10 = self.makefile(10)
3335 contents11, n11, newuri11 = self.makefile(11)
3338 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3341 "ctime": 1002777696.7564139,
3342 "mtime": 1002777696.7564139
3345 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3348 "ctime": 1002777696.7564139,
3349 "mtime": 1002777696.7564139
3352 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3355 "ctime": 1002777696.7564139,
3356 "mtime": 1002777696.7564139
3359 }""" % (newuri9, newuri10, newuri11)
3361 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3363 d = client.getPage(url, method="POST", postdata=reqbody)
3365 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3366 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3367 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3369 d.addCallback(_then)
3370 d.addErrback(self.dump_error)
3373 def test_POST_set_children_with_hyphen(self):
3374 return self.test_POST_set_children(command_name="set-children")
3376 def test_POST_link_uri(self):
3377 contents, n, newuri = self.makefile(8)
3378 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3379 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3380 d.addCallback(lambda res:
3381 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3385 def test_POST_link_uri_replace(self):
3386 contents, n, newuri = self.makefile(8)
3387 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3388 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3389 d.addCallback(lambda res:
3390 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3394 def test_POST_link_uri_unknown_bad(self):
3395 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3396 d.addBoth(self.shouldFail, error.Error,
3397 "POST_link_uri_unknown_bad",
3399 "unknown cap in a write slot")
3402 def test_POST_link_uri_unknown_ro_good(self):
3403 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3404 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3407 def test_POST_link_uri_unknown_imm_good(self):
3408 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3409 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3412 def test_POST_link_uri_no_replace_queryarg(self):
3413 contents, n, newuri = self.makefile(8)
3414 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3415 name="bar.txt", uri=newuri)
3416 d.addBoth(self.shouldFail, error.Error,
3417 "POST_link_uri_no_replace_queryarg",
3419 "There was already a child by that name, and you asked me "
3420 "to not replace it")
3421 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3422 d.addCallback(self.failUnlessIsBarDotTxt)
3425 def test_POST_link_uri_no_replace_field(self):
3426 contents, n, newuri = self.makefile(8)
3427 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3428 name="bar.txt", uri=newuri)
3429 d.addBoth(self.shouldFail, error.Error,
3430 "POST_link_uri_no_replace_field",
3432 "There was already a child by that name, and you asked me "
3433 "to not replace it")
3434 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3435 d.addCallback(self.failUnlessIsBarDotTxt)
3438 def test_POST_delete(self, command_name='delete'):
3439 d = self._foo_node.list()
3440 def _check_before(children):
3441 self.failUnlessIn(u"bar.txt", children)
3442 d.addCallback(_check_before)
3443 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3444 d.addCallback(lambda res: self._foo_node.list())
3445 def _check_after(children):
3446 self.failIfIn(u"bar.txt", children)
3447 d.addCallback(_check_after)
3450 def test_POST_unlink(self):
3451 return self.test_POST_delete(command_name='unlink')
3453 def test_POST_rename_file(self):
3454 d = self.POST(self.public_url + "/foo", t="rename",
3455 from_name="bar.txt", to_name='wibble.txt')
3456 d.addCallback(lambda res:
3457 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3458 d.addCallback(lambda res:
3459 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3460 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3461 d.addCallback(self.failUnlessIsBarDotTxt)
3462 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3463 d.addCallback(self.failUnlessIsBarJSON)
3466 def test_POST_rename_file_redundant(self):
3467 d = self.POST(self.public_url + "/foo", t="rename",
3468 from_name="bar.txt", to_name='bar.txt')
3469 d.addCallback(lambda res:
3470 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3471 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3472 d.addCallback(self.failUnlessIsBarDotTxt)
3473 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3474 d.addCallback(self.failUnlessIsBarJSON)
3477 def test_POST_rename_file_replace(self):
3478 # rename a file and replace a directory with it
3479 d = self.POST(self.public_url + "/foo", t="rename",
3480 from_name="bar.txt", to_name='empty')
3481 d.addCallback(lambda res:
3482 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3483 d.addCallback(lambda res:
3484 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3485 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3486 d.addCallback(self.failUnlessIsBarDotTxt)
3487 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3488 d.addCallback(self.failUnlessIsBarJSON)
3491 def test_POST_rename_file_no_replace_queryarg(self):
3492 # rename a file and replace a directory with it
3493 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3494 from_name="bar.txt", to_name='empty')
3495 d.addBoth(self.shouldFail, error.Error,
3496 "POST_rename_file_no_replace_queryarg",
3498 "There was already a child by that name, and you asked me "
3499 "to not replace it")
3500 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3501 d.addCallback(self.failUnlessIsEmptyJSON)
3504 def test_POST_rename_file_no_replace_field(self):
3505 # rename a file and replace a directory with it
3506 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3507 from_name="bar.txt", to_name='empty')
3508 d.addBoth(self.shouldFail, error.Error,
3509 "POST_rename_file_no_replace_field",
3511 "There was already a child by that name, and you asked me "
3512 "to not replace it")
3513 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3514 d.addCallback(self.failUnlessIsEmptyJSON)
3517 def test_POST_rename_file_no_replace_same_link(self):
3518 d = self.POST(self.public_url + "/foo", t="rename",
3519 replace="false", from_name="bar.txt", to_name="bar.txt")
3520 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3521 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3522 d.addCallback(self.failUnlessIsBarDotTxt)
3523 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3524 d.addCallback(self.failUnlessIsBarJSON)
3527 def test_POST_rename_file_replace_only_files(self):
3528 d = self.POST(self.public_url + "/foo", t="rename",
3529 replace="only-files", from_name="bar.txt",
3531 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3532 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3533 d.addCallback(self.failUnlessIsBarDotTxt)
3534 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3535 d.addCallback(self.failUnlessIsBarJSON)
3538 def test_POST_rename_file_replace_only_files_conflict(self):
3539 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3541 "There was already a child by that name, and you asked me to not replace it.",
3542 self.POST, self.public_url + "/foo", t="relink",
3543 replace="only-files", from_name="bar.txt",
3545 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3546 d.addCallback(self.failUnlessIsBarDotTxt)
3547 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3548 d.addCallback(self.failUnlessIsBarJSON)
3551 def failUnlessIsEmptyJSON(self, res):
3552 data = simplejson.loads(res)
3553 self.failUnlessEqual(data[0], "dirnode", data)
3554 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3556 def test_POST_rename_file_to_slash_fail(self):
3557 d = self.POST(self.public_url + "/foo", t="rename",
3558 from_name="bar.txt", to_name='kirk/spock.txt')
3559 d.addBoth(self.shouldFail, error.Error,
3560 "test_POST_rename_file_to_slash_fail",
3562 "to_name= may not contain a slash",
3564 d.addCallback(lambda res:
3565 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3568 def test_POST_rename_file_from_slash_fail(self):
3569 d = self.POST(self.public_url + "/foo", t="rename",
3570 from_name="sub/bar.txt", to_name='spock.txt')
3571 d.addBoth(self.shouldFail, error.Error,
3572 "test_POST_rename_from_file_slash_fail",
3574 "from_name= may not contain a slash",
3576 d.addCallback(lambda res:
3577 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3580 def test_POST_rename_dir(self):
3581 d = self.POST(self.public_url, t="rename",
3582 from_name="foo", to_name='plunk')
3583 d.addCallback(lambda res:
3584 self.failIfNodeHasChild(self.public_root, u"foo"))
3585 d.addCallback(lambda res:
3586 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3587 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3588 d.addCallback(self.failUnlessIsFooJSON)
3591 def test_POST_relink_file(self):
3592 d = self.POST(self.public_url + "/foo", t="relink",
3593 from_name="bar.txt",
3594 to_dir=self.public_root.get_uri() + "/foo/sub")
3595 d.addCallback(lambda res:
3596 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3597 d.addCallback(lambda res:
3598 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3599 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3600 d.addCallback(self.failUnlessIsBarDotTxt)
3601 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3602 d.addCallback(self.failUnlessIsBarJSON)
3605 def test_POST_relink_file_new_name(self):
3606 d = self.POST(self.public_url + "/foo", t="relink",
3607 from_name="bar.txt",
3608 to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3609 d.addCallback(lambda res:
3610 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3611 d.addCallback(lambda res:
3612 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3613 d.addCallback(lambda res:
3614 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3615 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3616 d.addCallback(self.failUnlessIsBarDotTxt)
3617 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3618 d.addCallback(self.failUnlessIsBarJSON)
3621 def test_POST_relink_file_replace(self):
3622 d = self.POST(self.public_url + "/foo", t="relink",
3623 from_name="bar.txt",
3624 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3625 d.addCallback(lambda res:
3626 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3627 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3628 d.addCallback(self.failUnlessIsBarDotTxt)
3629 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3630 d.addCallback(self.failUnlessIsBarJSON)
3633 def test_POST_relink_file_no_replace(self):
3634 d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3636 "There was already a child by that name, and you asked me to not replace it",
3637 self.POST, self.public_url + "/foo", t="relink",
3638 replace="false", from_name="bar.txt",
3639 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3640 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3641 d.addCallback(self.failUnlessIsBarDotTxt)
3642 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3643 d.addCallback(self.failUnlessIsBarJSON)
3644 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3645 d.addCallback(self.failUnlessIsSubBazDotTxt)
3648 def test_POST_relink_file_no_replace_explicitly_same_link(self):
3649 d = self.POST(self.public_url + "/foo", t="relink",
3650 replace="false", from_name="bar.txt",
3651 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3652 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3653 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3654 d.addCallback(self.failUnlessIsBarDotTxt)
3655 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3656 d.addCallback(self.failUnlessIsBarJSON)
3659 def test_POST_relink_file_replace_only_files(self):
3660 d = self.POST(self.public_url + "/foo", t="relink",
3661 replace="only-files", from_name="bar.txt",
3662 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3663 d.addCallback(lambda res:
3664 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3665 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3666 d.addCallback(self.failUnlessIsBarDotTxt)
3667 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3668 d.addCallback(self.failUnlessIsBarJSON)
3671 def test_POST_relink_file_replace_only_files_conflict(self):
3672 d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3674 "There was already a child by that name, and you asked me to not replace it.",
3675 self.POST, self.public_url + "/foo", t="relink",
3676 replace="only-files", from_name="bar.txt",
3677 to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3678 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3679 d.addCallback(self.failUnlessIsBarDotTxt)
3680 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3681 d.addCallback(self.failUnlessIsBarJSON)
3684 def test_POST_relink_file_to_slash_fail(self):
3685 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3687 "to_name= may not contain a slash",
3688 self.POST, self.public_url + "/foo", t="relink",
3689 from_name="bar.txt",
3690 to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3691 d.addCallback(lambda res:
3692 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3693 d.addCallback(lambda res:
3694 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3695 d.addCallback(lambda ign:
3696 self.shouldFail2(error.Error,
3697 "test_POST_rename_file_slash_fail2",
3699 "from_name= may not contain a slash",
3700 self.POST, self.public_url + "/foo",
3702 from_name="nope/bar.txt",
3704 to_dir=self.public_root.get_uri() + "/foo/sub"))
3707 def test_POST_relink_file_explicitly_same_link(self):
3708 d = self.POST(self.public_url + "/foo", t="relink",
3709 from_name="bar.txt",
3710 to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3711 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3712 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3713 d.addCallback(self.failUnlessIsBarDotTxt)
3714 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3715 d.addCallback(self.failUnlessIsBarJSON)
3718 def test_POST_relink_file_implicitly_same_link(self):
3719 d = self.POST(self.public_url + "/foo", t="relink",
3720 from_name="bar.txt")
3721 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3722 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3723 d.addCallback(self.failUnlessIsBarDotTxt)
3724 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3725 d.addCallback(self.failUnlessIsBarJSON)
3728 def test_POST_relink_file_same_dir(self):
3729 d = self.POST(self.public_url + "/foo", t="relink",
3730 from_name="bar.txt",
3731 to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3732 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3733 d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3734 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3735 d.addCallback(self.failUnlessIsBarDotTxt)
3736 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3737 d.addCallback(self.failUnlessIsBarJSON)
3740 def test_POST_relink_file_bad_replace(self):
3741 d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3742 "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3744 self.public_url + "/foo", t="relink",
3745 replace="boogabooga", from_name="bar.txt",
3746 to_dir=self.public_root.get_uri() + "/foo/sub")
3749 def test_POST_relink_file_multi_level(self):
3750 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3751 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3752 from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3753 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3754 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3755 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3756 d.addCallback(self.failUnlessIsBarDotTxt)
3757 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3758 d.addCallback(self.failUnlessIsBarJSON)
3761 def test_POST_relink_file_to_uri(self):
3762 d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3763 from_name="bar.txt", to_dir=self._sub_uri)
3764 d.addCallback(lambda res:
3765 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3766 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3767 d.addCallback(self.failUnlessIsBarDotTxt)
3768 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3769 d.addCallback(self.failUnlessIsBarJSON)
3772 def test_POST_relink_file_to_nonexistent_dir(self):
3773 d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3774 "404 Not Found", "No such child: nopechucktesta",
3775 self.POST, self.public_url + "/foo", t="relink",
3776 from_name="bar.txt",
3777 to_dir=self.public_root.get_uri() + "/nopechucktesta")
3780 def test_POST_relink_file_into_file(self):
3781 d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3782 "400 Bad Request", "to_dir is not a directory",
3783 self.POST, self.public_url + "/foo", t="relink",
3784 from_name="bar.txt",
3785 to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3786 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3787 d.addCallback(self.failUnlessIsBazDotTxt)
3788 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3789 d.addCallback(self.failUnlessIsBarDotTxt)
3790 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3791 d.addCallback(self.failUnlessIsBarJSON)
3794 def test_POST_relink_file_to_bad_uri(self):
3795 d = self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3796 "400 Bad Request", "to_dir is not a directory",
3797 self.POST, self.public_url + "/foo", t="relink",
3798 from_name="bar.txt",
3799 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3800 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3801 d.addCallback(self.failUnlessIsBarDotTxt)
3802 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3803 d.addCallback(self.failUnlessIsBarJSON)
3806 def test_POST_relink_dir(self):
3807 d = self.POST(self.public_url + "/foo", t="relink",
3808 from_name="bar.txt",
3809 to_dir=self.public_root.get_uri() + "/foo/empty")
3810 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3811 t="relink", from_name="empty",
3812 to_dir=self.public_root.get_uri() + "/foo/sub"))
3813 d.addCallback(lambda res:
3814 self.failIfNodeHasChild(self._foo_node, u"empty"))
3815 d.addCallback(lambda res:
3816 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3817 d.addCallback(lambda res:
3818 self._sub_node.get_child_at_path(u"empty"))
3819 d.addCallback(lambda node:
3820 self.failUnlessNodeHasChild(node, u"bar.txt"))
3821 d.addCallback(lambda res:
3822 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3823 d.addCallback(self.failUnlessIsBarDotTxt)
3826 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3827 """ If target is not None then the redirection has to go to target. If
3828 statuscode is not None then the redirection has to be accomplished with
3829 that HTTP status code."""
3830 if not isinstance(res, failure.Failure):
3831 to_where = (target is None) and "somewhere" or ("to " + target)
3832 self.fail("%s: we were expecting to get redirected %s, not get an"
3833 " actual page: %s" % (which, to_where, res))
3834 res.trap(error.PageRedirect)
3835 if statuscode is not None:
3836 self.failUnlessReallyEqual(res.value.status, statuscode,
3837 "%s: not a redirect" % which)
3838 if target is not None:
3839 # the PageRedirect does not seem to capture the uri= query arg
3840 # properly, so we can't check for it.
3841 realtarget = self.webish_url + target
3842 self.failUnlessReallyEqual(res.value.location, realtarget,
3843 "%s: wrong target" % which)
3844 return res.value.location
3846 def test_GET_URI_form(self):
3847 base = "/uri?uri=%s" % self._bar_txt_uri
3848 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3849 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3851 d.addBoth(self.shouldRedirect, targetbase)
3852 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3853 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3854 d.addCallback(lambda res: self.GET(base+"&t=json"))
3855 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3856 d.addCallback(self.log, "about to get file by uri")
3857 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3858 d.addCallback(self.failUnlessIsBarDotTxt)
3859 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3860 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3861 followRedirect=True))
3862 d.addCallback(self.failUnlessIsFooJSON)
3863 d.addCallback(self.log, "got dir by uri")
3867 def test_GET_URI_form_bad(self):
3868 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3869 "400 Bad Request", "GET /uri requires uri=",
3873 def test_GET_rename_form(self):
3874 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3875 followRedirect=True)
3877 self.failUnlessIn('name="when_done" value="."', res)
3878 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3879 self.failUnlessIn(FAVICON_MARKUP, res)
3880 d.addCallback(_check)
3883 def log(self, res, msg):
3884 #print "MSG: %s RES: %s" % (msg, res)
3888 def test_GET_URI_URL(self):
3889 base = "/uri/%s" % self._bar_txt_uri
3891 d.addCallback(self.failUnlessIsBarDotTxt)
3892 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3893 d.addCallback(self.failUnlessIsBarDotTxt)
3894 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3895 d.addCallback(self.failUnlessIsBarDotTxt)
3898 def test_GET_URI_URL_dir(self):
3899 base = "/uri/%s?t=json" % self._foo_uri
3901 d.addCallback(self.failUnlessIsFooJSON)
3904 def test_GET_URI_URL_missing(self):
3905 base = "/uri/%s" % self._bad_file_uri
3906 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3907 http.GONE, None, "NotEnoughSharesError",
3909 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3910 # here? we must arrange for a download to fail after target.open()
3911 # has been called, and then inspect the response to see that it is
3912 # shorter than we expected.
3915 def test_PUT_DIRURL_uri(self):
3916 d = self.s.create_dirnode()
3918 new_uri = dn.get_uri()
3919 # replace /foo with a new (empty) directory
3920 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3921 d.addCallback(lambda res:
3922 self.failUnlessReallyEqual(res.strip(), new_uri))
3923 d.addCallback(lambda res:
3924 self.failUnlessRWChildURIIs(self.public_root,
3928 d.addCallback(_made_dir)
3931 def test_PUT_DIRURL_uri_noreplace(self):
3932 d = self.s.create_dirnode()
3934 new_uri = dn.get_uri()
3935 # replace /foo with a new (empty) directory, but ask that
3936 # replace=false, so it should fail
3937 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3938 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3940 self.public_url + "/foo?t=uri&replace=false",
3942 d.addCallback(lambda res:
3943 self.failUnlessRWChildURIIs(self.public_root,
3947 d.addCallback(_made_dir)
3950 def test_PUT_DIRURL_bad_t(self):
3951 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3952 "400 Bad Request", "PUT to a directory",
3953 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3954 d.addCallback(lambda res:
3955 self.failUnlessRWChildURIIs(self.public_root,
3960 def test_PUT_NEWFILEURL_uri(self):
3961 contents, n, new_uri = self.makefile(8)
3962 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3963 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3964 d.addCallback(lambda res:
3965 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3969 def test_PUT_NEWFILEURL_mdmf(self):
3970 new_contents = self.NEWFILE_CONTENTS * 300000
3971 d = self.PUT(self.public_url + \
3972 "/foo/mdmf.txt?format=mdmf",
3974 d.addCallback(lambda ignored:
3975 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3976 def _got_json(json):
3977 data = simplejson.loads(json)
3979 self.failUnlessIn("format", data)
3980 self.failUnlessEqual(data["format"], "MDMF")
3981 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3982 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3983 d.addCallback(_got_json)
3986 def test_PUT_NEWFILEURL_sdmf(self):
3987 new_contents = self.NEWFILE_CONTENTS * 300000
3988 d = self.PUT(self.public_url + \
3989 "/foo/sdmf.txt?format=sdmf",
3991 d.addCallback(lambda ignored:
3992 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3993 def _got_json(json):
3994 data = simplejson.loads(json)
3996 self.failUnlessIn("format", data)
3997 self.failUnlessEqual(data["format"], "SDMF")
3998 d.addCallback(_got_json)
4001 def test_PUT_NEWFILEURL_bad_format(self):
4002 new_contents = self.NEWFILE_CONTENTS * 300000
4003 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4004 400, "Bad Request", "Unknown format: foo",
4005 self.PUT, self.public_url + \
4006 "/foo/foo.txt?format=foo",
4009 def test_PUT_NEWFILEURL_uri_replace(self):
4010 contents, n, new_uri = self.makefile(8)
4011 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4012 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4013 d.addCallback(lambda res:
4014 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4018 def test_PUT_NEWFILEURL_uri_no_replace(self):
4019 contents, n, new_uri = self.makefile(8)
4020 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4021 d.addBoth(self.shouldFail, error.Error,
4022 "PUT_NEWFILEURL_uri_no_replace",
4024 "There was already a child by that name, and you asked me "
4025 "to not replace it")
4028 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4029 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4030 d.addBoth(self.shouldFail, error.Error,
4031 "POST_put_uri_unknown_bad",
4033 "unknown cap in a write slot")
4036 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4037 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4038 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4039 u"put-future-ro.txt")
4042 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4043 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4044 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4045 u"put-future-imm.txt")
4048 def test_PUT_NEWFILE_URI(self):
4049 file_contents = "New file contents here\n"
4050 d = self.PUT("/uri", file_contents)
4052 assert isinstance(uri, str), uri
4053 self.failUnlessIn(uri, self.get_all_contents())
4054 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4056 return self.GET("/uri/%s" % uri)
4057 d.addCallback(_check)
4059 self.failUnlessReallyEqual(res, file_contents)
4060 d.addCallback(_check2)
4063 def test_PUT_NEWFILE_URI_not_mutable(self):
4064 file_contents = "New file contents here\n"
4065 d = self.PUT("/uri?mutable=false", file_contents)
4067 assert isinstance(uri, str), uri
4068 self.failUnlessIn(uri, self.get_all_contents())
4069 self.failUnlessReallyEqual(self.get_all_contents()[uri],
4071 return self.GET("/uri/%s" % uri)
4072 d.addCallback(_check)
4074 self.failUnlessReallyEqual(res, file_contents)
4075 d.addCallback(_check2)
4078 def test_PUT_NEWFILE_URI_only_PUT(self):
4079 d = self.PUT("/uri?t=bogus", "")
4080 d.addBoth(self.shouldFail, error.Error,
4081 "PUT_NEWFILE_URI_only_PUT",
4083 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4086 def test_PUT_NEWFILE_URI_mutable(self):
4087 file_contents = "New file contents here\n"
4088 d = self.PUT("/uri?mutable=true", file_contents)
4089 def _check1(filecap):
4090 filecap = filecap.strip()
4091 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4092 self.filecap = filecap
4093 u = uri.WriteableSSKFileURI.init_from_string(filecap)
4094 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4095 n = self.s.create_node_from_uri(filecap)
4096 return n.download_best_version()
4097 d.addCallback(_check1)
4099 self.failUnlessReallyEqual(data, file_contents)
4100 return self.GET("/uri/%s" % urllib.quote(self.filecap))
4101 d.addCallback(_check2)
4103 self.failUnlessReallyEqual(res, file_contents)
4104 d.addCallback(_check3)
4107 def test_PUT_mkdir(self):
4108 d = self.PUT("/uri?t=mkdir", "")
4110 n = self.s.create_node_from_uri(uri.strip())
4111 d2 = self.failUnlessNodeKeysAre(n, [])
4112 d2.addCallback(lambda res:
4113 self.GET("/uri/%s?t=json" % uri))
4115 d.addCallback(_check)
4116 d.addCallback(self.failUnlessIsEmptyJSON)
4119 def test_PUT_mkdir_mdmf(self):
4120 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4122 u = uri.from_string(res)
4123 # Check that this is an MDMF writecap
4124 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4128 def test_PUT_mkdir_sdmf(self):
4129 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4131 u = uri.from_string(res)
4132 self.failUnlessIsInstance(u, uri.DirectoryURI)
4136 def test_PUT_mkdir_bad_format(self):
4137 return self.shouldHTTPError("PUT_mkdir_bad_format",
4138 400, "Bad Request", "Unknown format: foo",
4139 self.PUT, "/uri?t=mkdir&format=foo",
4142 def test_POST_check(self):
4143 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4145 # this returns a string form of the results, which are probably
4146 # None since we're using fake filenodes.
4147 # TODO: verify that the check actually happened, by changing
4148 # FakeCHKFileNode to count how many times .check() has been
4151 d.addCallback(_done)
4155 def test_PUT_update_at_offset(self):
4156 file_contents = "test file" * 100000 # about 900 KiB
4157 d = self.PUT("/uri?mutable=true", file_contents)
4159 self.filecap = filecap
4160 new_data = file_contents[:100]
4161 new = "replaced and so on"
4163 new_data += file_contents[len(new_data):]
4164 assert len(new_data) == len(file_contents)
4165 self.new_data = new_data
4166 d.addCallback(_then)
4167 d.addCallback(lambda ignored:
4168 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4169 "replaced and so on"))
4170 def _get_data(filecap):
4171 n = self.s.create_node_from_uri(filecap)
4172 return n.download_best_version()
4173 d.addCallback(_get_data)
4174 d.addCallback(lambda results:
4175 self.failUnlessEqual(results, self.new_data))
4176 # Now try appending things to the file
4177 d.addCallback(lambda ignored:
4178 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4180 d.addCallback(_get_data)
4181 d.addCallback(lambda results:
4182 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4183 # and try replacing the beginning of the file
4184 d.addCallback(lambda ignored:
4185 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4186 d.addCallback(_get_data)
4187 d.addCallback(lambda results:
4188 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4191 def test_PUT_update_at_invalid_offset(self):
4192 file_contents = "test file" * 100000 # about 900 KiB
4193 d = self.PUT("/uri?mutable=true", file_contents)
4195 self.filecap = filecap
4196 d.addCallback(_then)
4197 # Negative offsets should cause an error.
4198 d.addCallback(lambda ignored:
4199 self.shouldHTTPError("PUT_update_at_invalid_offset",
4203 "/uri/%s?offset=-1" % self.filecap,
4207 def test_PUT_update_at_offset_immutable(self):
4208 file_contents = "Test file" * 100000
4209 d = self.PUT("/uri", file_contents)
4211 self.filecap = filecap
4212 d.addCallback(_then)
4213 d.addCallback(lambda ignored:
4214 self.shouldHTTPError("PUT_update_at_offset_immutable",
4218 "/uri/%s?offset=50" % self.filecap,
4223 def test_bad_method(self):
4224 url = self.webish_url + self.public_url + "/foo/bar.txt"
4225 d = self.shouldHTTPError("bad_method",
4226 501, "Not Implemented",
4227 "I don't know how to treat a BOGUS request.",
4228 client.getPage, url, method="BOGUS")
4231 def test_short_url(self):
4232 url = self.webish_url + "/uri"
4233 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4234 "I don't know how to treat a DELETE request.",
4235 client.getPage, url, method="DELETE")
4238 def test_ophandle_bad(self):
4239 url = self.webish_url + "/operations/bogus?t=status"
4240 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4241 "unknown/expired handle 'bogus'",
4242 client.getPage, url)
4245 def test_ophandle_cancel(self):
4246 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4247 followRedirect=True)
4248 d.addCallback(lambda ignored:
4249 self.GET("/operations/128?t=status&output=JSON"))
4251 data = simplejson.loads(res)
4252 self.failUnless("finished" in data, res)
4253 monitor = self.ws.root.child_operations.handles["128"][0]
4254 d = self.POST("/operations/128?t=cancel&output=JSON")
4256 data = simplejson.loads(res)
4257 self.failUnless("finished" in data, res)
4258 # t=cancel causes the handle to be forgotten
4259 self.failUnless(monitor.is_cancelled())
4260 d.addCallback(_check2)
4262 d.addCallback(_check1)
4263 d.addCallback(lambda ignored:
4264 self.shouldHTTPError("ophandle_cancel",
4265 404, "404 Not Found",
4266 "unknown/expired handle '128'",
4268 "/operations/128?t=status&output=JSON"))
4271 def test_ophandle_retainfor(self):
4272 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4273 followRedirect=True)
4274 d.addCallback(lambda ignored:
4275 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4277 data = simplejson.loads(res)
4278 self.failUnless("finished" in data, res)
4279 d.addCallback(_check1)
4280 # the retain-for=0 will cause the handle to be expired very soon
4281 d.addCallback(lambda ign:
4282 self.clock.advance(2.0))
4283 d.addCallback(lambda ignored:
4284 self.shouldHTTPError("ophandle_retainfor",
4285 404, "404 Not Found",
4286 "unknown/expired handle '129'",
4288 "/operations/129?t=status&output=JSON"))
4291 def test_ophandle_release_after_complete(self):
4292 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4293 followRedirect=True)
4294 d.addCallback(self.wait_for_operation, "130")
4295 d.addCallback(lambda ignored:
4296 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4297 # the release-after-complete=true will cause the handle to be expired
4298 d.addCallback(lambda ignored:
4299 self.shouldHTTPError("ophandle_release_after_complete",
4300 404, "404 Not Found",
4301 "unknown/expired handle '130'",
4303 "/operations/130?t=status&output=JSON"))
4306 def test_uncollected_ophandle_expiration(self):
4307 # uncollected ophandles should expire after 4 days
4308 def _make_uncollected_ophandle(ophandle):
4309 d = self.POST(self.public_url +
4310 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4311 followRedirect=False)
4312 # When we start the operation, the webapi server will want
4313 # to redirect us to the page for the ophandle, so we get
4314 # confirmation that the operation has started. If the
4315 # manifest operation has finished by the time we get there,
4316 # following that redirect (by setting followRedirect=True
4317 # above) has the side effect of collecting the ophandle that
4318 # we've just created, which means that we can't use the
4319 # ophandle to test the uncollected timeout anymore. So,
4320 # instead, catch the 302 here and don't follow it.
4321 d.addBoth(self.should302, "uncollected_ophandle_creation")
4323 # Create an ophandle, don't collect it, then advance the clock by
4324 # 4 days - 1 second and make sure that the ophandle is still there.
4325 d = _make_uncollected_ophandle(131)
4326 d.addCallback(lambda ign:
4327 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4328 d.addCallback(lambda ign:
4329 self.GET("/operations/131?t=status&output=JSON"))
4331 data = simplejson.loads(res)
4332 self.failUnless("finished" in data, res)
4333 d.addCallback(_check1)
4334 # Create an ophandle, don't collect it, then try to collect it
4335 # after 4 days. It should be gone.
4336 d.addCallback(lambda ign:
4337 _make_uncollected_ophandle(132))
4338 d.addCallback(lambda ign:
4339 self.clock.advance(96*60*60))
4340 d.addCallback(lambda ign:
4341 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4342 404, "404 Not Found",
4343 "unknown/expired handle '132'",
4345 "/operations/132?t=status&output=JSON"))
4348 def test_collected_ophandle_expiration(self):
4349 # collected ophandles should expire after 1 day
4350 def _make_collected_ophandle(ophandle):
4351 d = self.POST(self.public_url +
4352 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4353 followRedirect=True)
4354 # By following the initial redirect, we collect the ophandle
4355 # we've just created.
4357 # Create a collected ophandle, then collect it after 23 hours
4358 # and 59 seconds to make sure that it is still there.
4359 d = _make_collected_ophandle(133)
4360 d.addCallback(lambda ign:
4361 self.clock.advance((24*60*60) - 1))
4362 d.addCallback(lambda ign:
4363 self.GET("/operations/133?t=status&output=JSON"))
4365 data = simplejson.loads(res)
4366 self.failUnless("finished" in data, res)
4367 d.addCallback(_check1)
4368 # Create another uncollected ophandle, then try to collect it
4369 # after 24 hours to make sure that it is gone.
4370 d.addCallback(lambda ign:
4371 _make_collected_ophandle(134))
4372 d.addCallback(lambda ign:
4373 self.clock.advance(24*60*60))
4374 d.addCallback(lambda ign:
4375 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4376 404, "404 Not Found",
4377 "unknown/expired handle '134'",
4379 "/operations/134?t=status&output=JSON"))
4382 def test_incident(self):
4383 d = self.POST("/report_incident", details="eek")
4385 self.failIfIn("<html>", res)
4386 self.failUnlessIn("An incident report has been saved", res)
4387 d.addCallback(_done)
4390 def test_static(self):
4391 webdir = os.path.join(self.staticdir, "subdir")
4392 fileutil.make_dirs(webdir)
4393 f = open(os.path.join(webdir, "hello.txt"), "wb")
4397 d = self.GET("/static/subdir/hello.txt")
4399 self.failUnlessReallyEqual(res, "hello")
4400 d.addCallback(_check)
4404 class IntroducerWeb(unittest.TestCase):
4409 d = defer.succeed(None)
4411 d.addCallback(lambda ign: self.node.stopService())
4412 d.addCallback(flushEventualQueue)
4415 def test_welcome(self):
4416 basedir = "web.IntroducerWeb.test_welcome"
4418 cfg = "\n".join(["[node]",
4419 "tub.location = 127.0.0.1:1",
4422 fileutil.write(os.path.join(basedir, "tahoe.cfg"), cfg)
4423 self.node = IntroducerNode(basedir)
4424 self.ws = self.node.getServiceNamed("webish")
4426 d = fireEventually(None)
4427 d.addCallback(lambda ign: self.node.startService())
4428 d.addCallback(lambda ign: self.node.when_tub_ready())
4430 d.addCallback(lambda ign: self.GET("/"))
4432 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4433 self.failUnlessIn(FAVICON_MARKUP, res)
4434 self.failUnlessIn('Page rendered at', res)
4435 self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4436 d.addCallback(_check)
4439 def GET(self, urlpath, followRedirect=False, return_response=False,
4441 # if return_response=True, this fires with (data, statuscode,
4442 # respheaders) instead of just data.
4443 assert not isinstance(urlpath, unicode)
4444 url = self.ws.getURL().rstrip('/') + urlpath
4445 factory = HTTPClientGETFactory(url, method="GET",
4446 followRedirect=followRedirect, **kwargs)
4447 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4448 d = factory.deferred
4449 def _got_data(data):
4450 return (data, factory.status, factory.response_headers)
4452 d.addCallback(_got_data)
4453 return factory.deferred
4456 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4457 def test_load_file(self):
4458 # This will raise an exception unless a well-formed XML file is found under that name.
4459 common.getxmlfile('directory.xhtml').load()
4461 def test_parse_replace_arg(self):
4462 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4463 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4464 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4466 self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4468 def test_abbreviate_time(self):
4469 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4470 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4471 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4472 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4473 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4474 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4476 def test_compute_rate(self):
4477 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4478 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4479 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4480 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4481 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4482 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4483 self.shouldFail(AssertionError, "test_compute_rate", "",
4484 common.compute_rate, -100, 10)
4485 self.shouldFail(AssertionError, "test_compute_rate", "",
4486 common.compute_rate, 100, -10)
4489 rate = common.compute_rate(10*1000*1000, 1)
4490 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4492 def test_abbreviate_rate(self):
4493 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4494 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4495 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4496 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4498 def test_abbreviate_size(self):
4499 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4500 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4501 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4502 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4503 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4505 def test_plural(self):
4507 return "%d second%s" % (s, status.plural(s))
4508 self.failUnlessReallyEqual(convert(0), "0 seconds")
4509 self.failUnlessReallyEqual(convert(1), "1 second")
4510 self.failUnlessReallyEqual(convert(2), "2 seconds")
4512 return "has share%s: %s" % (status.plural(s), ",".join(s))
4513 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4514 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4515 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4518 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4520 def CHECK(self, ign, which, args, clientnum=0):
4521 fileurl = self.fileurls[which]
4522 url = fileurl + "?" + args
4523 return self.GET(url, method="POST", clientnum=clientnum)
4525 def test_filecheck(self):
4526 self.basedir = "web/Grid/filecheck"
4528 c0 = self.g.clients[0]
4531 d = c0.upload(upload.Data(DATA, convergence=""))
4532 def _stash_uri(ur, which):
4533 self.uris[which] = ur.get_uri()
4534 d.addCallback(_stash_uri, "good")
4535 d.addCallback(lambda ign:
4536 c0.upload(upload.Data(DATA+"1", convergence="")))
4537 d.addCallback(_stash_uri, "sick")
4538 d.addCallback(lambda ign:
4539 c0.upload(upload.Data(DATA+"2", convergence="")))
4540 d.addCallback(_stash_uri, "dead")
4541 def _stash_mutable_uri(n, which):
4542 self.uris[which] = n.get_uri()
4543 assert isinstance(self.uris[which], str)
4544 d.addCallback(lambda ign:
4545 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4546 d.addCallback(_stash_mutable_uri, "corrupt")
4547 d.addCallback(lambda ign:
4548 c0.upload(upload.Data("literal", convergence="")))
4549 d.addCallback(_stash_uri, "small")
4550 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4551 d.addCallback(_stash_mutable_uri, "smalldir")
4553 def _compute_fileurls(ignored):
4555 for which in self.uris:
4556 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4557 d.addCallback(_compute_fileurls)
4559 def _clobber_shares(ignored):
4560 good_shares = self.find_uri_shares(self.uris["good"])
4561 self.failUnlessReallyEqual(len(good_shares), 10)
4562 sick_shares = self.find_uri_shares(self.uris["sick"])
4563 os.unlink(sick_shares[0][2])
4564 dead_shares = self.find_uri_shares(self.uris["dead"])
4565 for i in range(1, 10):
4566 os.unlink(dead_shares[i][2])
4567 c_shares = self.find_uri_shares(self.uris["corrupt"])
4568 cso = CorruptShareOptions()
4569 cso.stdout = StringIO()
4570 cso.parseOptions([c_shares[0][2]])
4572 d.addCallback(_clobber_shares)
4574 d.addCallback(self.CHECK, "good", "t=check")
4575 def _got_html_good(res):
4576 self.failUnlessIn("Healthy", res)
4577 self.failIfIn("Not Healthy", res)
4578 self.failUnlessIn(FAVICON_MARKUP, res)
4579 d.addCallback(_got_html_good)
4580 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4581 def _got_html_good_return_to(res):
4582 self.failUnlessIn("Healthy", res)
4583 self.failIfIn("Not Healthy", res)
4584 self.failUnlessIn('<a href="somewhere">Return to file', res)
4585 d.addCallback(_got_html_good_return_to)
4586 d.addCallback(self.CHECK, "good", "t=check&output=json")
4587 def _got_json_good(res):
4588 r = simplejson.loads(res)
4589 self.failUnlessEqual(r["summary"], "Healthy")
4590 self.failUnless(r["results"]["healthy"])
4591 self.failIfIn("needs-rebalancing", r["results"])
4592 self.failUnless(r["results"]["recoverable"])
4593 d.addCallback(_got_json_good)
4595 d.addCallback(self.CHECK, "small", "t=check")
4596 def _got_html_small(res):
4597 self.failUnlessIn("Literal files are always healthy", res)
4598 self.failIfIn("Not Healthy", res)
4599 d.addCallback(_got_html_small)
4600 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4601 def _got_html_small_return_to(res):
4602 self.failUnlessIn("Literal files are always healthy", res)
4603 self.failIfIn("Not Healthy", res)
4604 self.failUnlessIn('<a href="somewhere">Return to file', res)
4605 d.addCallback(_got_html_small_return_to)
4606 d.addCallback(self.CHECK, "small", "t=check&output=json")
4607 def _got_json_small(res):
4608 r = simplejson.loads(res)
4609 self.failUnlessEqual(r["storage-index"], "")
4610 self.failUnless(r["results"]["healthy"])
4611 d.addCallback(_got_json_small)
4613 d.addCallback(self.CHECK, "smalldir", "t=check")
4614 def _got_html_smalldir(res):
4615 self.failUnlessIn("Literal files are always healthy", res)
4616 self.failIfIn("Not Healthy", res)
4617 d.addCallback(_got_html_smalldir)
4618 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4619 def _got_json_smalldir(res):
4620 r = simplejson.loads(res)
4621 self.failUnlessEqual(r["storage-index"], "")
4622 self.failUnless(r["results"]["healthy"])
4623 d.addCallback(_got_json_smalldir)
4625 d.addCallback(self.CHECK, "sick", "t=check")
4626 def _got_html_sick(res):
4627 self.failUnlessIn("Not Healthy", res)
4628 d.addCallback(_got_html_sick)
4629 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4630 def _got_json_sick(res):
4631 r = simplejson.loads(res)
4632 self.failUnlessEqual(r["summary"],
4633 "Not Healthy: 9 shares (enc 3-of-10)")
4634 self.failIf(r["results"]["healthy"])
4635 self.failUnless(r["results"]["recoverable"])
4636 self.failIfIn("needs-rebalancing", r["results"])
4637 d.addCallback(_got_json_sick)
4639 d.addCallback(self.CHECK, "dead", "t=check")
4640 def _got_html_dead(res):
4641 self.failUnlessIn("Not Healthy", res)
4642 d.addCallback(_got_html_dead)
4643 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4644 def _got_json_dead(res):
4645 r = simplejson.loads(res)
4646 self.failUnlessEqual(r["summary"],
4647 "Not Healthy: 1 shares (enc 3-of-10)")
4648 self.failIf(r["results"]["healthy"])
4649 self.failIf(r["results"]["recoverable"])
4650 self.failIfIn("needs-rebalancing", r["results"])
4651 d.addCallback(_got_json_dead)
4653 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4654 def _got_html_corrupt(res):
4655 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4656 d.addCallback(_got_html_corrupt)
4657 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4658 def _got_json_corrupt(res):
4659 r = simplejson.loads(res)
4660 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4661 self.failIf(r["results"]["healthy"])
4662 self.failUnless(r["results"]["recoverable"])
4663 self.failIfIn("needs-rebalancing", r["results"])
4664 self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4665 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4666 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4667 d.addCallback(_got_json_corrupt)
4669 d.addErrback(self.explain_web_error)
4672 def test_repair_html(self):
4673 self.basedir = "web/Grid/repair_html"
4675 c0 = self.g.clients[0]
4678 d = c0.upload(upload.Data(DATA, convergence=""))
4679 def _stash_uri(ur, which):
4680 self.uris[which] = ur.get_uri()
4681 d.addCallback(_stash_uri, "good")
4682 d.addCallback(lambda ign:
4683 c0.upload(upload.Data(DATA+"1", convergence="")))
4684 d.addCallback(_stash_uri, "sick")
4685 d.addCallback(lambda ign:
4686 c0.upload(upload.Data(DATA+"2", convergence="")))
4687 d.addCallback(_stash_uri, "dead")
4688 def _stash_mutable_uri(n, which):
4689 self.uris[which] = n.get_uri()
4690 assert isinstance(self.uris[which], str)
4691 d.addCallback(lambda ign:
4692 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4693 d.addCallback(_stash_mutable_uri, "corrupt")
4695 def _compute_fileurls(ignored):
4697 for which in self.uris:
4698 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4699 d.addCallback(_compute_fileurls)
4701 def _clobber_shares(ignored):
4702 good_shares = self.find_uri_shares(self.uris["good"])
4703 self.failUnlessReallyEqual(len(good_shares), 10)
4704 sick_shares = self.find_uri_shares(self.uris["sick"])
4705 os.unlink(sick_shares[0][2])
4706 dead_shares = self.find_uri_shares(self.uris["dead"])
4707 for i in range(1, 10):
4708 os.unlink(dead_shares[i][2])
4709 c_shares = self.find_uri_shares(self.uris["corrupt"])
4710 cso = CorruptShareOptions()
4711 cso.stdout = StringIO()
4712 cso.parseOptions([c_shares[0][2]])
4714 d.addCallback(_clobber_shares)
4716 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4717 def _got_html_good(res):
4718 self.failUnlessIn("Healthy", res)
4719 self.failIfIn("Not Healthy", res)
4720 self.failUnlessIn("No repair necessary", res)
4721 self.failUnlessIn(FAVICON_MARKUP, res)
4722 d.addCallback(_got_html_good)
4724 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4725 def _got_html_sick(res):
4726 self.failUnlessIn("Healthy : healthy", res)
4727 self.failIfIn("Not Healthy", res)
4728 self.failUnlessIn("Repair successful", res)
4729 d.addCallback(_got_html_sick)
4731 # repair of a dead file will fail, of course, but it isn't yet
4732 # clear how this should be reported. Right now it shows up as
4735 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4736 #def _got_html_dead(res):
4738 # self.failUnlessIn("Healthy : healthy", res)
4739 # self.failIfIn("Not Healthy", res)
4740 # self.failUnlessIn("No repair necessary", res)
4741 #d.addCallback(_got_html_dead)
4743 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4744 def _got_html_corrupt(res):
4745 self.failUnlessIn("Healthy : Healthy", res)
4746 self.failIfIn("Not Healthy", res)
4747 self.failUnlessIn("Repair successful", res)
4748 d.addCallback(_got_html_corrupt)
4750 d.addErrback(self.explain_web_error)
4753 def test_repair_json(self):
4754 self.basedir = "web/Grid/repair_json"
4756 c0 = self.g.clients[0]
4759 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4760 def _stash_uri(ur, which):
4761 self.uris[which] = ur.get_uri()
4762 d.addCallback(_stash_uri, "sick")
4764 def _compute_fileurls(ignored):
4766 for which in self.uris:
4767 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4768 d.addCallback(_compute_fileurls)
4770 def _clobber_shares(ignored):
4771 sick_shares = self.find_uri_shares(self.uris["sick"])
4772 os.unlink(sick_shares[0][2])
4773 d.addCallback(_clobber_shares)
4775 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4776 def _got_json_sick(res):
4777 r = simplejson.loads(res)
4778 self.failUnlessReallyEqual(r["repair-attempted"], True)
4779 self.failUnlessReallyEqual(r["repair-successful"], True)
4780 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4781 "Not Healthy: 9 shares (enc 3-of-10)")
4782 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4783 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4784 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4785 d.addCallback(_got_json_sick)
4787 d.addErrback(self.explain_web_error)
4790 def test_unknown(self, immutable=False):
4791 self.basedir = "web/Grid/unknown"
4793 self.basedir = "web/Grid/unknown-immutable"
4796 c0 = self.g.clients[0]
4800 # the future cap format may contain slashes, which must be tolerated
4801 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4805 name = u"future-imm"
4806 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4807 d = c0.create_immutable_dirnode({name: (future_node, {})})
4810 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4811 d = c0.create_dirnode()
4813 def _stash_root_and_create_file(n):
4815 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4816 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4818 return self.rootnode.set_node(name, future_node)
4819 d.addCallback(_stash_root_and_create_file)
4821 # make sure directory listing tolerates unknown nodes
4822 d.addCallback(lambda ign: self.GET(self.rooturl))
4823 def _check_directory_html(res, expected_type_suffix):
4824 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4825 '<td>%s</td>' % (expected_type_suffix, str(name)),
4827 self.failUnless(re.search(pattern, res), res)
4828 # find the More Info link for name, should be relative
4829 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4830 info_url = mo.group(1)
4831 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4833 d.addCallback(_check_directory_html, "-IMM")
4835 d.addCallback(_check_directory_html, "")
4837 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4838 def _check_directory_json(res, expect_rw_uri):
4839 data = simplejson.loads(res)
4840 self.failUnlessEqual(data[0], "dirnode")
4841 f = data[1]["children"][name]
4842 self.failUnlessEqual(f[0], "unknown")
4844 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4846 self.failIfIn("rw_uri", f[1])
4848 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4850 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4851 self.failUnlessIn("metadata", f[1])
4852 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4854 def _check_info(res, expect_rw_uri, expect_ro_uri):
4855 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4857 self.failUnlessIn(unknown_rwcap, res)
4860 self.failUnlessIn(unknown_immcap, res)
4862 self.failUnlessIn(unknown_rocap, res)
4864 self.failIfIn(unknown_rocap, res)
4865 self.failIfIn("Raw data as", res)
4866 self.failIfIn("Directory writecap", res)
4867 self.failIfIn("Checker Operations", res)
4868 self.failIfIn("Mutable File Operations", res)
4869 self.failIfIn("Directory Operations", res)
4871 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4872 # why they fail. Possibly related to ticket #922.
4874 d.addCallback(lambda ign: self.GET(expected_info_url))
4875 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4876 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4877 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4879 def _check_json(res, expect_rw_uri):
4880 data = simplejson.loads(res)
4881 self.failUnlessEqual(data[0], "unknown")
4883 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4885 self.failIfIn("rw_uri", data[1])
4888 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4889 self.failUnlessReallyEqual(data[1]["mutable"], False)
4891 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4892 self.failUnlessReallyEqual(data[1]["mutable"], True)
4894 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4895 self.failIfIn("mutable", data[1])
4897 # TODO: check metadata contents
4898 self.failUnlessIn("metadata", data[1])
4900 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4901 d.addCallback(_check_json, expect_rw_uri=not immutable)
4903 # and make sure that a read-only version of the directory can be
4904 # rendered too. This version will not have unknown_rwcap, whether
4905 # or not future_node was immutable.
4906 d.addCallback(lambda ign: self.GET(self.rourl))
4908 d.addCallback(_check_directory_html, "-IMM")
4910 d.addCallback(_check_directory_html, "-RO")
4912 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4913 d.addCallback(_check_directory_json, expect_rw_uri=False)
4915 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4916 d.addCallback(_check_json, expect_rw_uri=False)
4918 # TODO: check that getting t=info from the Info link in the ro directory
4919 # works, and does not include the writecap URI.
4922 def test_immutable_unknown(self):
4923 return self.test_unknown(immutable=True)
4925 def test_mutant_dirnodes_are_omitted(self):
4926 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4929 c = self.g.clients[0]
4934 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4935 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4936 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4938 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4939 # test the dirnode and web layers separately.
4941 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4942 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4943 # When the directory is read, the mutants should be silently disposed of, leaving
4944 # their lonely sibling.
4945 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4946 # because immutable directories don't have a writecap and therefore that field
4947 # isn't (and can't be) decrypted.
4948 # TODO: The field still exists in the netstring. Technically we should check what
4949 # happens if something is put there (_unpack_contents should raise ValueError),
4950 # but that can wait.
4952 lonely_child = nm.create_from_cap(lonely_uri)
4953 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4954 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4956 def _by_hook_or_by_crook():
4958 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4959 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4961 mutant_write_in_ro_child.get_write_uri = lambda: None
4962 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4964 kids = {u"lonely": (lonely_child, {}),
4965 u"ro": (mutant_ro_child, {}),
4966 u"write-in-ro": (mutant_write_in_ro_child, {}),
4968 d = c.create_immutable_dirnode(kids)
4971 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4972 self.failIf(dn.is_mutable())
4973 self.failUnless(dn.is_readonly())
4974 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4975 self.failIf(hasattr(dn._node, 'get_writekey'))
4977 self.failUnlessIn("RO-IMM", rep)
4979 self.failUnlessIn("CHK", cap.to_string())
4982 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4983 return download_to_data(dn._node)
4984 d.addCallback(_created)
4986 def _check_data(data):
4987 # Decode the netstring representation of the directory to check that all children
4988 # are present. This is a bit of an abstraction violation, but there's not really
4989 # any other way to do it given that the real DirectoryNode._unpack_contents would
4990 # strip the mutant children out (which is what we're trying to test, later).
4993 while position < len(data):
4994 entries, position = split_netstring(data, 1, position)
4996 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4997 name = name_utf8.decode("utf-8")
4998 self.failUnlessEqual(rwcapdata, "")
4999 self.failUnlessIn(name, kids)
5000 (expected_child, ign) = kids[name]
5001 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
5004 self.failUnlessReallyEqual(numkids, 3)
5005 return self.rootnode.list()
5006 d.addCallback(_check_data)
5008 # Now when we use the real directory listing code, the mutants should be absent.
5009 def _check_kids(children):
5010 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5011 lonely_node, lonely_metadata = children[u"lonely"]
5013 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5014 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5015 d.addCallback(_check_kids)
5017 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5018 d.addCallback(lambda n: n.list())
5019 d.addCallback(_check_kids) # again with dirnode recreated from cap
5021 # Make sure the lonely child can be listed in HTML...
5022 d.addCallback(lambda ign: self.GET(self.rooturl))
5023 def _check_html(res):
5024 self.failIfIn("URI:SSK", res)
5025 get_lonely = "".join([r'<td>FILE</td>',
5027 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5029 r'\s+<td align="right">%d</td>' % len("one"),
5031 self.failUnless(re.search(get_lonely, res), res)
5033 # find the More Info link for name, should be relative
5034 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5035 info_url = mo.group(1)
5036 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5037 d.addCallback(_check_html)
5040 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5041 def _check_json(res):
5042 data = simplejson.loads(res)
5043 self.failUnlessEqual(data[0], "dirnode")
5044 listed_children = data[1]["children"]
5045 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5046 ll_type, ll_data = listed_children[u"lonely"]
5047 self.failUnlessEqual(ll_type, "filenode")
5048 self.failIfIn("rw_uri", ll_data)
5049 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5050 d.addCallback(_check_json)
5053 def test_deep_check(self):
5054 self.basedir = "web/Grid/deep_check"
5056 c0 = self.g.clients[0]
5060 d = c0.create_dirnode()
5061 def _stash_root_and_create_file(n):
5063 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5064 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5065 d.addCallback(_stash_root_and_create_file)
5066 def _stash_uri(fn, which):
5067 self.uris[which] = fn.get_uri()
5069 d.addCallback(_stash_uri, "good")
5070 d.addCallback(lambda ign:
5071 self.rootnode.add_file(u"small",
5072 upload.Data("literal",
5074 d.addCallback(_stash_uri, "small")
5075 d.addCallback(lambda ign:
5076 self.rootnode.add_file(u"sick",
5077 upload.Data(DATA+"1",
5079 d.addCallback(_stash_uri, "sick")
5081 # this tests that deep-check and stream-manifest will ignore
5082 # UnknownNode instances. Hopefully this will also cover deep-stats.
5083 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5084 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5086 def _clobber_shares(ignored):
5087 self.delete_shares_numbered(self.uris["sick"], [0,1])
5088 d.addCallback(_clobber_shares)
5096 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5099 units = [simplejson.loads(line)
5100 for line in res.splitlines()
5103 print "response is:", res
5104 print "undecodeable line was '%s'" % line
5106 self.failUnlessReallyEqual(len(units), 5+1)
5107 # should be parent-first
5109 self.failUnlessEqual(u0["path"], [])
5110 self.failUnlessEqual(u0["type"], "directory")
5111 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5112 u0cr = u0["check-results"]
5113 self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5114 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5116 ugood = [u for u in units
5117 if u["type"] == "file" and u["path"] == [u"good"]][0]
5118 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5119 ugoodcr = ugood["check-results"]
5120 self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5121 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5124 self.failUnlessEqual(stats["type"], "stats")
5126 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5127 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5128 self.failUnlessReallyEqual(s["count-directories"], 1)
5129 self.failUnlessReallyEqual(s["count-unknown"], 1)
5130 d.addCallback(_done)
5132 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5133 def _check_manifest(res):
5134 self.failUnless(res.endswith("\n"))
5135 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5136 self.failUnlessReallyEqual(len(units), 5+1)
5137 self.failUnlessEqual(units[-1]["type"], "stats")
5139 self.failUnlessEqual(first["path"], [])
5140 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5141 self.failUnlessEqual(first["type"], "directory")
5142 stats = units[-1]["stats"]
5143 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5144 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5145 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5146 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5147 self.failUnlessReallyEqual(stats["count-unknown"], 1)
5148 d.addCallback(_check_manifest)
5150 # now add root/subdir and root/subdir/grandchild, then make subdir
5151 # unrecoverable, then see what happens
5153 d.addCallback(lambda ign:
5154 self.rootnode.create_subdirectory(u"subdir"))
5155 d.addCallback(_stash_uri, "subdir")
5156 d.addCallback(lambda subdir_node:
5157 subdir_node.add_file(u"grandchild",
5158 upload.Data(DATA+"2",
5160 d.addCallback(_stash_uri, "grandchild")
5162 d.addCallback(lambda ign:
5163 self.delete_shares_numbered(self.uris["subdir"],
5171 # root/subdir [unrecoverable]
5172 # root/subdir/grandchild
5174 # how should a streaming-JSON API indicate fatal error?
5175 # answer: emit ERROR: instead of a JSON string
5177 d.addCallback(self.CHECK, "root", "t=stream-manifest")
5178 def _check_broken_manifest(res):
5179 lines = res.splitlines()
5181 for (i,line) in enumerate(lines)
5182 if line.startswith("ERROR:")]
5184 self.fail("no ERROR: in output: %s" % (res,))
5185 first_error = error_lines[0]
5186 error_line = lines[first_error]
5187 error_msg = lines[first_error+1:]
5188 error_msg_s = "\n".join(error_msg) + "\n"
5189 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5191 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5192 units = [simplejson.loads(line) for line in lines[:first_error]]
5193 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5194 last_unit = units[-1]
5195 self.failUnlessEqual(last_unit["path"], ["subdir"])
5196 d.addCallback(_check_broken_manifest)
5198 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5199 def _check_broken_deepcheck(res):
5200 lines = res.splitlines()
5202 for (i,line) in enumerate(lines)
5203 if line.startswith("ERROR:")]
5205 self.fail("no ERROR: in output: %s" % (res,))
5206 first_error = error_lines[0]
5207 error_line = lines[first_error]
5208 error_msg = lines[first_error+1:]
5209 error_msg_s = "\n".join(error_msg) + "\n"
5210 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5212 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5213 units = [simplejson.loads(line) for line in lines[:first_error]]
5214 self.failUnlessReallyEqual(len(units), 6) # includes subdir
5215 last_unit = units[-1]
5216 self.failUnlessEqual(last_unit["path"], ["subdir"])
5217 r = last_unit["check-results"]["results"]
5218 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5219 self.failUnlessReallyEqual(r["count-happiness"], 1)
5220 self.failUnlessReallyEqual(r["count-shares-good"], 1)
5221 self.failUnlessReallyEqual(r["recoverable"], False)
5222 d.addCallback(_check_broken_deepcheck)
5224 d.addErrback(self.explain_web_error)
5227 def test_deep_check_and_repair(self):
5228 self.basedir = "web/Grid/deep_check_and_repair"
5230 c0 = self.g.clients[0]
5234 d = c0.create_dirnode()
5235 def _stash_root_and_create_file(n):
5237 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5238 return n.add_file(u"good", upload.Data(DATA, convergence=""))
5239 d.addCallback(_stash_root_and_create_file)
5240 def _stash_uri(fn, which):
5241 self.uris[which] = fn.get_uri()
5242 d.addCallback(_stash_uri, "good")
5243 d.addCallback(lambda ign:
5244 self.rootnode.add_file(u"small",
5245 upload.Data("literal",
5247 d.addCallback(_stash_uri, "small")
5248 d.addCallback(lambda ign:
5249 self.rootnode.add_file(u"sick",
5250 upload.Data(DATA+"1",
5252 d.addCallback(_stash_uri, "sick")
5253 #d.addCallback(lambda ign:
5254 # self.rootnode.add_file(u"dead",
5255 # upload.Data(DATA+"2",
5257 #d.addCallback(_stash_uri, "dead")
5259 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5260 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5261 #d.addCallback(_stash_uri, "corrupt")
5263 def _clobber_shares(ignored):
5264 good_shares = self.find_uri_shares(self.uris["good"])
5265 self.failUnlessReallyEqual(len(good_shares), 10)
5266 sick_shares = self.find_uri_shares(self.uris["sick"])
5267 os.unlink(sick_shares[0][2])
5268 #dead_shares = self.find_uri_shares(self.uris["dead"])
5269 #for i in range(1, 10):
5270 # os.unlink(dead_shares[i][2])
5272 #c_shares = self.find_uri_shares(self.uris["corrupt"])
5273 #cso = CorruptShareOptions()
5274 #cso.stdout = StringIO()
5275 #cso.parseOptions([c_shares[0][2]])
5277 d.addCallback(_clobber_shares)
5280 # root/good CHK, 10 shares
5282 # root/sick CHK, 9 shares
5284 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5286 units = [simplejson.loads(line)
5287 for line in res.splitlines()
5289 self.failUnlessReallyEqual(len(units), 4+1)
5290 # should be parent-first
5292 self.failUnlessEqual(u0["path"], [])
5293 self.failUnlessEqual(u0["type"], "directory")
5294 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5295 u0crr = u0["check-and-repair-results"]
5296 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5297 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5298 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5300 ugood = [u for u in units
5301 if u["type"] == "file" and u["path"] == [u"good"]][0]
5302 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5303 ugoodcrr = ugood["check-and-repair-results"]
5304 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5305 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5306 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5308 usick = [u for u in units
5309 if u["type"] == "file" and u["path"] == [u"sick"]][0]
5310 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5311 usickcrr = usick["check-and-repair-results"]
5312 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5313 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5314 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5315 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5316 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5317 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5320 self.failUnlessEqual(stats["type"], "stats")
5322 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5323 self.failUnlessReallyEqual(s["count-literal-files"], 1)
5324 self.failUnlessReallyEqual(s["count-directories"], 1)
5325 d.addCallback(_done)
5327 d.addErrback(self.explain_web_error)
5330 def _count_leases(self, ignored, which):
5331 u = self.uris[which]
5332 shares = self.find_uri_shares(u)
5334 for shnum, serverid, fn in shares:
5335 sf = get_share_file(fn)
5336 num_leases = len(list(sf.get_leases()))
5337 lease_counts.append( (fn, num_leases) )
5340 def _assert_leasecount(self, lease_counts, expected):
5341 for (fn, num_leases) in lease_counts:
5342 if num_leases != expected:
5343 self.fail("expected %d leases, have %d, on %s" %
5344 (expected, num_leases, fn))
5346 def test_add_lease(self):
5347 self.basedir = "web/Grid/add_lease"
5348 self.set_up_grid(num_clients=2)
5349 c0 = self.g.clients[0]
5352 d = c0.upload(upload.Data(DATA, convergence=""))
5353 def _stash_uri(ur, which):
5354 self.uris[which] = ur.get_uri()
5355 d.addCallback(_stash_uri, "one")
5356 d.addCallback(lambda ign:
5357 c0.upload(upload.Data(DATA+"1", convergence="")))
5358 d.addCallback(_stash_uri, "two")
5359 def _stash_mutable_uri(n, which):
5360 self.uris[which] = n.get_uri()
5361 assert isinstance(self.uris[which], str)
5362 d.addCallback(lambda ign:
5363 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5364 d.addCallback(_stash_mutable_uri, "mutable")
5366 def _compute_fileurls(ignored):
5368 for which in self.uris:
5369 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5370 d.addCallback(_compute_fileurls)
5372 d.addCallback(self._count_leases, "one")
5373 d.addCallback(self._assert_leasecount, 1)
5374 d.addCallback(self._count_leases, "two")
5375 d.addCallback(self._assert_leasecount, 1)
5376 d.addCallback(self._count_leases, "mutable")
5377 d.addCallback(self._assert_leasecount, 1)
5379 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5380 def _got_html_good(res):
5381 self.failUnlessIn("Healthy", res)
5382 self.failIfIn("Not Healthy", res)
5383 d.addCallback(_got_html_good)
5385 d.addCallback(self._count_leases, "one")
5386 d.addCallback(self._assert_leasecount, 1)
5387 d.addCallback(self._count_leases, "two")
5388 d.addCallback(self._assert_leasecount, 1)
5389 d.addCallback(self._count_leases, "mutable")
5390 d.addCallback(self._assert_leasecount, 1)
5392 # this CHECK uses the original client, which uses the same
5393 # lease-secrets, so it will just renew the original lease
5394 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5395 d.addCallback(_got_html_good)
5397 d.addCallback(self._count_leases, "one")
5398 d.addCallback(self._assert_leasecount, 1)
5399 d.addCallback(self._count_leases, "two")
5400 d.addCallback(self._assert_leasecount, 1)
5401 d.addCallback(self._count_leases, "mutable")
5402 d.addCallback(self._assert_leasecount, 1)
5404 # this CHECK uses an alternate client, which adds a second lease
5405 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5406 d.addCallback(_got_html_good)
5408 d.addCallback(self._count_leases, "one")
5409 d.addCallback(self._assert_leasecount, 2)
5410 d.addCallback(self._count_leases, "two")
5411 d.addCallback(self._assert_leasecount, 1)
5412 d.addCallback(self._count_leases, "mutable")
5413 d.addCallback(self._assert_leasecount, 1)
5415 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5416 d.addCallback(_got_html_good)
5418 d.addCallback(self._count_leases, "one")
5419 d.addCallback(self._assert_leasecount, 2)
5420 d.addCallback(self._count_leases, "two")
5421 d.addCallback(self._assert_leasecount, 1)
5422 d.addCallback(self._count_leases, "mutable")
5423 d.addCallback(self._assert_leasecount, 1)
5425 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5427 d.addCallback(_got_html_good)
5429 d.addCallback(self._count_leases, "one")
5430 d.addCallback(self._assert_leasecount, 2)
5431 d.addCallback(self._count_leases, "two")
5432 d.addCallback(self._assert_leasecount, 1)
5433 d.addCallback(self._count_leases, "mutable")
5434 d.addCallback(self._assert_leasecount, 2)
5436 d.addErrback(self.explain_web_error)
5439 def test_deep_add_lease(self):
5440 self.basedir = "web/Grid/deep_add_lease"
5441 self.set_up_grid(num_clients=2)
5442 c0 = self.g.clients[0]
5446 d = c0.create_dirnode()
5447 def _stash_root_and_create_file(n):
5449 self.uris["root"] = n.get_uri()
5450 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5451 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5452 d.addCallback(_stash_root_and_create_file)
5453 def _stash_uri(fn, which):
5454 self.uris[which] = fn.get_uri()
5455 d.addCallback(_stash_uri, "one")
5456 d.addCallback(lambda ign:
5457 self.rootnode.add_file(u"small",
5458 upload.Data("literal",
5460 d.addCallback(_stash_uri, "small")
5462 d.addCallback(lambda ign:
5463 c0.create_mutable_file(publish.MutableData("mutable")))
5464 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5465 d.addCallback(_stash_uri, "mutable")
5467 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5469 units = [simplejson.loads(line)
5470 for line in res.splitlines()
5472 # root, one, small, mutable, stats
5473 self.failUnlessReallyEqual(len(units), 4+1)
5474 d.addCallback(_done)
5476 d.addCallback(self._count_leases, "root")
5477 d.addCallback(self._assert_leasecount, 1)
5478 d.addCallback(self._count_leases, "one")
5479 d.addCallback(self._assert_leasecount, 1)
5480 d.addCallback(self._count_leases, "mutable")
5481 d.addCallback(self._assert_leasecount, 1)
5483 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5484 d.addCallback(_done)
5486 d.addCallback(self._count_leases, "root")
5487 d.addCallback(self._assert_leasecount, 1)
5488 d.addCallback(self._count_leases, "one")
5489 d.addCallback(self._assert_leasecount, 1)
5490 d.addCallback(self._count_leases, "mutable")
5491 d.addCallback(self._assert_leasecount, 1)
5493 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5495 d.addCallback(_done)
5497 d.addCallback(self._count_leases, "root")
5498 d.addCallback(self._assert_leasecount, 2)
5499 d.addCallback(self._count_leases, "one")
5500 d.addCallback(self._assert_leasecount, 2)
5501 d.addCallback(self._count_leases, "mutable")
5502 d.addCallback(self._assert_leasecount, 2)
5504 d.addErrback(self.explain_web_error)
5508 def test_exceptions(self):
5509 self.basedir = "web/Grid/exceptions"
5510 self.set_up_grid(num_clients=1, num_servers=2)
5511 c0 = self.g.clients[0]
5512 c0.encoding_params['happy'] = 2
5515 d = c0.create_dirnode()
5517 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5518 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5520 d.addCallback(_stash_root)
5521 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5523 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5524 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5526 u = uri.from_string(ur.get_uri())
5527 u.key = testutil.flip_bit(u.key, 0)
5528 baduri = u.to_string()
5529 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5530 d.addCallback(_stash_bad)
5531 d.addCallback(lambda ign: c0.create_dirnode())
5532 def _mangle_dirnode_1share(n):
5534 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5535 self.fileurls["dir-1share-json"] = url + "?t=json"
5536 self.delete_shares_numbered(u, range(1,10))
5537 d.addCallback(_mangle_dirnode_1share)
5538 d.addCallback(lambda ign: c0.create_dirnode())
5539 def _mangle_dirnode_0share(n):
5541 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5542 self.fileurls["dir-0share-json"] = url + "?t=json"
5543 self.delete_shares_numbered(u, range(0,10))
5544 d.addCallback(_mangle_dirnode_0share)
5546 # NotEnoughSharesError should be reported sensibly, with a
5547 # text/plain explanation of the problem, and perhaps some
5548 # information on which shares *could* be found.
5550 d.addCallback(lambda ignored:
5551 self.shouldHTTPError("GET unrecoverable",
5552 410, "Gone", "NoSharesError",
5553 self.GET, self.fileurls["0shares"]))
5554 def _check_zero_shares(body):
5555 self.failIfIn("<html>", body)
5556 body = " ".join(body.strip().split())
5557 exp = ("NoSharesError: no shares could be found. "
5558 "Zero shares usually indicates a corrupt URI, or that "
5559 "no servers were connected, but it might also indicate "
5560 "severe corruption. You should perform a filecheck on "
5561 "this object to learn more. The full error message is: "
5562 "no shares (need 3). Last failure: None")
5563 self.failUnlessReallyEqual(exp, body)
5564 d.addCallback(_check_zero_shares)
5567 d.addCallback(lambda ignored:
5568 self.shouldHTTPError("GET 1share",
5569 410, "Gone", "NotEnoughSharesError",
5570 self.GET, self.fileurls["1share"]))
5571 def _check_one_share(body):
5572 self.failIfIn("<html>", body)
5573 body = " ".join(body.strip().split())
5574 msgbase = ("NotEnoughSharesError: This indicates that some "
5575 "servers were unavailable, or that shares have been "
5576 "lost to server departure, hard drive failure, or disk "
5577 "corruption. You should perform a filecheck on "
5578 "this object to learn more. The full error message is:"
5580 msg1 = msgbase + (" ran out of shares:"
5583 " overdue= unused= need 3. Last failure: None")
5584 msg2 = msgbase + (" ran out of shares:"
5586 " pending=Share(sh0-on-xgru5)"
5587 " overdue= unused= need 3. Last failure: None")
5588 self.failUnless(body == msg1 or body == msg2, body)
5589 d.addCallback(_check_one_share)
5591 d.addCallback(lambda ignored:
5592 self.shouldHTTPError("GET imaginary",
5593 404, "Not Found", None,
5594 self.GET, self.fileurls["imaginary"]))
5595 def _missing_child(body):
5596 self.failUnlessIn("No such child: imaginary", body)
5597 d.addCallback(_missing_child)
5599 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5600 def _check_0shares_dir_html(body):
5601 self.failUnlessIn(DIR_HTML_TAG, body)
5602 # we should see the regular page, but without the child table or
5604 body = " ".join(body.strip().split())
5605 self.failUnlessIn('href="?t=info">More info on this directory',
5607 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5608 "could not be retrieved, because there were insufficient "
5609 "good shares. This might indicate that no servers were "
5610 "connected, insufficient servers were connected, the URI "
5611 "was corrupt, or that shares have been lost due to server "
5612 "departure, hard drive failure, or disk corruption. You "
5613 "should perform a filecheck on this object to learn more.")
5614 self.failUnlessIn(exp, body)
5615 self.failUnlessIn("No upload forms: directory is unreadable", body)
5616 d.addCallback(_check_0shares_dir_html)
5618 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5619 def _check_1shares_dir_html(body):
5620 # at some point, we'll split UnrecoverableFileError into 0-shares
5621 # and some-shares like we did for immutable files (since there
5622 # are different sorts of advice to offer in each case). For now,
5623 # they present the same way.
5624 self.failUnlessIn(DIR_HTML_TAG, body)
5625 body = " ".join(body.strip().split())
5626 self.failUnlessIn('href="?t=info">More info on this directory',
5628 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5629 "could not be retrieved, because there were insufficient "
5630 "good shares. This might indicate that no servers were "
5631 "connected, insufficient servers were connected, the URI "
5632 "was corrupt, or that shares have been lost due to server "
5633 "departure, hard drive failure, or disk corruption. You "
5634 "should perform a filecheck on this object to learn more.")
5635 self.failUnlessIn(exp, body)
5636 self.failUnlessIn("No upload forms: directory is unreadable", body)
5637 d.addCallback(_check_1shares_dir_html)
5639 d.addCallback(lambda ignored:
5640 self.shouldHTTPError("GET dir-0share-json",
5641 410, "Gone", "UnrecoverableFileError",
5643 self.fileurls["dir-0share-json"]))
5644 def _check_unrecoverable_file(body):
5645 self.failIfIn("<html>", body)
5646 body = " ".join(body.strip().split())
5647 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5648 "could not be retrieved, because there were insufficient "
5649 "good shares. This might indicate that no servers were "
5650 "connected, insufficient servers were connected, the URI "
5651 "was corrupt, or that shares have been lost due to server "
5652 "departure, hard drive failure, or disk corruption. You "
5653 "should perform a filecheck on this object to learn more.")
5654 self.failUnlessReallyEqual(exp, body)
5655 d.addCallback(_check_unrecoverable_file)
5657 d.addCallback(lambda ignored:
5658 self.shouldHTTPError("GET dir-1share-json",
5659 410, "Gone", "UnrecoverableFileError",
5661 self.fileurls["dir-1share-json"]))
5662 d.addCallback(_check_unrecoverable_file)
5664 d.addCallback(lambda ignored:
5665 self.shouldHTTPError("GET imaginary",
5666 404, "Not Found", None,
5667 self.GET, self.fileurls["imaginary"]))
5669 # attach a webapi child that throws a random error, to test how it
5671 w = c0.getServiceNamed("webish")
5672 w.root.putChild("ERRORBOOM", ErrorBoom())
5674 # "Accept: */*" : should get a text/html stack trace
5675 # "Accept: text/plain" : should get a text/plain stack trace
5676 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5677 # no Accept header: should get a text/html stack trace
5679 d.addCallback(lambda ignored:
5680 self.shouldHTTPError("GET errorboom_html",
5681 500, "Internal Server Error", None,
5682 self.GET, "ERRORBOOM",
5683 headers={"accept": "*/*"}))
5684 def _internal_error_html1(body):
5685 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5686 d.addCallback(_internal_error_html1)
5688 d.addCallback(lambda ignored:
5689 self.shouldHTTPError("GET errorboom_text",
5690 500, "Internal Server Error", None,
5691 self.GET, "ERRORBOOM",
5692 headers={"accept": "text/plain"}))
5693 def _internal_error_text2(body):
5694 self.failIfIn("<html>", body)
5695 self.failUnless(body.startswith("Traceback "), body)
5696 d.addCallback(_internal_error_text2)
5698 CLI_accepts = "text/plain, application/octet-stream"
5699 d.addCallback(lambda ignored:
5700 self.shouldHTTPError("GET errorboom_text",
5701 500, "Internal Server Error", None,
5702 self.GET, "ERRORBOOM",
5703 headers={"accept": CLI_accepts}))
5704 def _internal_error_text3(body):
5705 self.failIfIn("<html>", body)
5706 self.failUnless(body.startswith("Traceback "), body)
5707 d.addCallback(_internal_error_text3)
5709 d.addCallback(lambda ignored:
5710 self.shouldHTTPError("GET errorboom_text",
5711 500, "Internal Server Error", None,
5712 self.GET, "ERRORBOOM"))
5713 def _internal_error_html4(body):
5714 self.failUnlessIn("<html>", body)
5715 d.addCallback(_internal_error_html4)
5717 def _flush_errors(res):
5718 # Trial: please ignore the CompletelyUnhandledError in the logs
5719 self.flushLoggedErrors(CompletelyUnhandledError)
5721 d.addBoth(_flush_errors)
5725 def test_blacklist(self):
5726 # download from a blacklisted URI, get an error
5727 self.basedir = "web/Grid/blacklist"
5729 c0 = self.g.clients[0]
5730 c0_basedir = c0.basedir
5731 fn = os.path.join(c0_basedir, "access.blacklist")
5733 DATA = "off-limits " * 50
5735 d = c0.upload(upload.Data(DATA, convergence=""))
5736 def _stash_uri_and_create_dir(ur):
5737 self.uri = ur.get_uri()
5738 self.url = "uri/"+self.uri
5739 u = uri.from_string_filenode(self.uri)
5740 self.si = u.get_storage_index()
5741 childnode = c0.create_node_from_uri(self.uri, None)
5742 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5743 d.addCallback(_stash_uri_and_create_dir)
5744 def _stash_dir(node):
5745 self.dir_node = node
5746 self.dir_uri = node.get_uri()
5747 self.dir_url = "uri/"+self.dir_uri
5748 d.addCallback(_stash_dir)
5749 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5750 def _check_dir_html(body):
5751 self.failUnlessIn(DIR_HTML_TAG, body)
5752 self.failUnlessIn("blacklisted.txt</a>", body)
5753 d.addCallback(_check_dir_html)
5754 d.addCallback(lambda ign: self.GET(self.url))
5755 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5757 def _blacklist(ign):
5759 f.write(" # this is a comment\n")
5761 f.write("\n") # also exercise blank lines
5762 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5764 # clients should be checking the blacklist each time, so we don't
5765 # need to restart the client
5766 d.addCallback(_blacklist)
5767 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5769 "Access Prohibited: off-limits",
5770 self.GET, self.url))
5772 # We should still be able to list the parent directory, in HTML...
5773 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5774 def _check_dir_html2(body):
5775 self.failUnlessIn(DIR_HTML_TAG, body)
5776 self.failUnlessIn("blacklisted.txt</strike>", body)
5777 d.addCallback(_check_dir_html2)
5779 # ... and in JSON (used by CLI).
5780 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5781 def _check_dir_json(res):
5782 data = simplejson.loads(res)
5783 self.failUnless(isinstance(data, list), data)
5784 self.failUnlessEqual(data[0], "dirnode")
5785 self.failUnless(isinstance(data[1], dict), data)
5786 self.failUnlessIn("children", data[1])
5787 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5788 childdata = data[1]["children"]["blacklisted.txt"]
5789 self.failUnless(isinstance(childdata, list), data)
5790 self.failUnlessEqual(childdata[0], "filenode")
5791 self.failUnless(isinstance(childdata[1], dict), data)
5792 d.addCallback(_check_dir_json)
5794 def _unblacklist(ign):
5795 open(fn, "w").close()
5796 # the Blacklist object watches mtime to tell when the file has
5797 # changed, but on windows this test will run faster than the
5798 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5799 # to force a reload.
5800 self.g.clients[0].blacklist.last_mtime -= 2.0
5801 d.addCallback(_unblacklist)
5803 # now a read should work
5804 d.addCallback(lambda ign: self.GET(self.url))
5805 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5807 # read again to exercise the blacklist-is-unchanged logic
5808 d.addCallback(lambda ign: self.GET(self.url))
5809 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5811 # now add a blacklisted directory, and make sure files under it are
5814 childnode = c0.create_node_from_uri(self.uri, None)
5815 return c0.create_dirnode({u"child": (childnode,{}) })
5816 d.addCallback(_add_dir)
5817 def _get_dircap(dn):
5818 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5819 self.dir_url_base = "uri/"+dn.get_write_uri()
5820 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5821 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5822 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5823 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5824 d.addCallback(_get_dircap)
5825 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5826 d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5827 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5828 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5829 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5830 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5831 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5832 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5833 d.addCallback(lambda ign: self.GET(self.child_url))
5834 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5836 def _block_dir(ign):
5838 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5840 self.g.clients[0].blacklist.last_mtime -= 2.0
5841 d.addCallback(_block_dir)
5842 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5844 "Access Prohibited: dir-off-limits",
5845 self.GET, self.dir_url_base))
5846 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5848 "Access Prohibited: dir-off-limits",
5849 self.GET, self.dir_url_json1))
5850 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5852 "Access Prohibited: dir-off-limits",
5853 self.GET, self.dir_url_json2))
5854 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5856 "Access Prohibited: dir-off-limits",
5857 self.GET, self.dir_url_json_ro))
5858 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5860 "Access Prohibited: dir-off-limits",
5861 self.GET, self.child_url))
5865 class CompletelyUnhandledError(Exception):
5867 class ErrorBoom(rend.Page):
5868 def beforeRender(self, ctx):
5869 raise CompletelyUnhandledError("whoops")