1 import os.path, re, urllib, time
3 from StringIO import StringIO
5 from twisted.application import service
6 from twisted.trial import unittest
7 from twisted.internet import defer, reactor
8 from twisted.internet.task import Clock
9 from twisted.web import client, error, http
10 from twisted.python import failure, log
12 from foolscap.api import fireEventually, flushEventualQueue
14 from nevow import rend
16 from allmydata import interfaces, uri, webish, dirnode
17 from allmydata.storage.shares import get_share_file
18 from allmydata.storage_client import StorageFarmBroker
19 from allmydata.immutable import upload
20 from allmydata.immutable.downloader.status import DownloadStatus
21 from allmydata.dirnode import DirectoryNode
22 from allmydata.nodemaker import NodeMaker
23 from allmydata.unknown import UnknownNode
24 from allmydata.web import status, common
25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
26 from allmydata.util import fileutil, base32, hashutil
27 from allmydata.util.consumer import download_to_data
28 from allmydata.util.netstring import split_netstring
29 from allmydata.util.encodingutil import to_str
30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
31 create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
32 make_mutable_file_uri, create_mutable_filenode
33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
34 from allmydata.mutable import servermap, publish, retrieve
35 import allmydata.test.common_util as testutil
36 from allmydata.test.no_network import GridTestMixin
37 from allmydata.test.common_web import HTTPClientGETFactory, \
39 from allmydata.client import Client, SecretHolder
40 from allmydata.introducer import IntroducerNode
42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
43 # create a webserver that works against them
45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
54 class FakeStatsProvider:
56 stats = {'stats': {}, 'counters': {}}
59 class FakeNodeMaker(NodeMaker):
64 'max_segment_size':128*1024 # 1024=KiB
66 def _create_lit(self, cap):
67 return FakeCHKFileNode(cap, self.all_contents)
68 def _create_immutable(self, cap):
69 return FakeCHKFileNode(cap, self.all_contents)
70 def _create_mutable(self, cap):
71 return FakeMutableFileNode(None, None,
72 self.encoding_params, None,
73 self.all_contents).init_from_cap(cap)
74 def create_mutable_file(self, contents="", keysize=None,
75 version=SDMF_VERSION):
76 n = FakeMutableFileNode(None, None, self.encoding_params, None,
78 return n.create(contents, version=version)
80 class FakeUploader(service.Service):
82 def upload(self, uploadable):
83 d = uploadable.get_size()
84 d.addCallback(lambda size: uploadable.read(size))
87 n = create_chk_filenode(data, self.all_contents)
88 ur = upload.UploadResults(file_size=len(data),
95 uri_extension_data={},
96 uri_extension_hash="fake",
97 verifycapstr="fakevcap")
98 ur.set_uri(n.get_uri())
100 d.addCallback(_got_data)
102 def get_helper_info(self):
106 def __init__(self, binaryserverid):
107 self.binaryserverid = binaryserverid
108 def get_name(self): return "short"
109 def get_longname(self): return "long"
110 def get_serverid(self): return self.binaryserverid
113 ds = DownloadStatus("storage_index", 1234)
116 serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
117 serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
118 storage_index = hashutil.storage_index_hash("SI")
119 e0 = ds.add_segment_request(0, now)
121 e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
122 e1 = ds.add_segment_request(1, now+2)
124 # two outstanding requests
125 e2 = ds.add_segment_request(2, now+4)
126 e3 = ds.add_segment_request(3, now+5)
127 del e2,e3 # hush pyflakes
129 # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
130 e = ds.add_segment_request(4, now)
132 e.deliver(now, 0, 140, 0.5)
134 e = ds.add_dyhb_request(serverA, now)
135 e.finished([1,2], now+1)
136 e = ds.add_dyhb_request(serverB, now+2) # left unfinished
138 e = ds.add_read_event(0, 120, now)
139 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
141 e = ds.add_read_event(120, 30, now+2) # left unfinished
143 e = ds.add_block_request(serverA, 1, 100, 20, now)
144 e.finished(20, now+1)
145 e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
147 # make sure that add_read_event() can come first too
148 ds1 = DownloadStatus(storage_index, 1234)
149 e = ds1.add_read_event(0, 120, now)
150 e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
156 _all_upload_status = [upload.UploadStatus()]
157 _all_download_status = [build_one_ds()]
158 _all_mapupdate_statuses = [servermap.UpdateStatus()]
159 _all_publish_statuses = [publish.PublishStatus()]
160 _all_retrieve_statuses = [retrieve.RetrieveStatus()]
162 def list_all_upload_statuses(self):
163 return self._all_upload_status
164 def list_all_download_statuses(self):
165 return self._all_download_status
166 def list_all_mapupdate_statuses(self):
167 return self._all_mapupdate_statuses
168 def list_all_publish_statuses(self):
169 return self._all_publish_statuses
170 def list_all_retrieve_statuses(self):
171 return self._all_retrieve_statuses
172 def list_all_helper_statuses(self):
175 class FakeClient(Client):
177 # don't upcall to Client.__init__, since we only want to initialize a
179 service.MultiService.__init__(self)
180 self.all_contents = {}
181 self.nodeid = "fake_nodeid"
182 self.nickname = "fake_nickname"
183 self.introducer_furl = "None"
184 self.stats_provider = FakeStatsProvider()
185 self._secret_holder = SecretHolder("lease secret", "convergence secret")
187 self.convergence = "some random string"
188 self.storage_broker = StorageFarmBroker(None, permute_peers=True)
189 self.introducer_client = None
190 self.history = FakeHistory()
191 self.uploader = FakeUploader()
192 self.uploader.all_contents = self.all_contents
193 self.uploader.setServiceParent(self)
194 self.blacklist = None
195 self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
198 self.nodemaker.all_contents = self.all_contents
199 self.mutable_file_default = SDMF_VERSION
201 def startService(self):
202 return service.MultiService.startService(self)
203 def stopService(self):
204 return service.MultiService.stopService(self)
206 MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
208 class WebMixin(object):
210 self.s = FakeClient()
211 self.s.startService()
212 self.staticdir = self.mktemp()
214 self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
216 self.ws.setServiceParent(self.s)
217 self.webish_port = self.ws.getPortnum()
218 self.webish_url = self.ws.getURL()
219 assert self.webish_url.endswith("/")
220 self.webish_url = self.webish_url[:-1] # these tests add their own /
222 l = [ self.s.create_dirnode() for x in range(6) ]
223 d = defer.DeferredList(l)
225 self.public_root = res[0][1]
226 assert interfaces.IDirectoryNode.providedBy(self.public_root), res
227 self.public_url = "/uri/" + self.public_root.get_uri()
228 self.private_root = res[1][1]
232 self._foo_uri = foo.get_uri()
233 self._foo_readonly_uri = foo.get_readonly_uri()
234 self._foo_verifycap = foo.get_verify_cap().to_string()
235 # NOTE: we ignore the deferred on all set_uri() calls, because we
236 # know the fake nodes do these synchronously
237 self.public_root.set_uri(u"foo", foo.get_uri(),
238 foo.get_readonly_uri())
240 self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
241 foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
242 self._bar_txt_verifycap = n.get_verify_cap().to_string()
245 # XXX: Do we ever use this?
246 self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
248 foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
251 self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
252 assert self._quux_txt_uri.startswith("URI:MDMF")
253 foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
255 foo.set_uri(u"empty", res[3][1].get_uri(),
256 res[3][1].get_readonly_uri())
257 sub_uri = res[4][1].get_uri()
258 self._sub_uri = sub_uri
259 foo.set_uri(u"sub", sub_uri, sub_uri)
260 sub = self.s.create_node_from_uri(sub_uri)
263 _ign, n, blocking_uri = self.makefile(1)
264 foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
266 unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
267 # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
268 # still think of it as an umlaut
269 foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
271 self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
272 self._baz_file_uri = baz_file
273 sub.set_uri(u"baz.txt", baz_file, baz_file)
275 _ign, n, self._bad_file_uri = self.makefile(3)
276 # this uri should not be downloadable
277 del self.s.all_contents[self._bad_file_uri]
280 self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
281 rodir.get_readonly_uri())
282 rodir.set_uri(u"nor", baz_file, baz_file)
288 # public/foo/quux.txt
289 # public/foo/blockingfile
292 # public/foo/sub/baz.txt
294 # public/reedownlee/nor
295 self.NEWFILE_CONTENTS = "newfile contents\n"
297 return foo.get_metadata_for(u"bar.txt")
299 def _got_metadata(metadata):
300 self._bar_txt_metadata = metadata
301 d.addCallback(_got_metadata)
304 def get_all_contents(self):
305 return self.s.all_contents
307 def makefile(self, number):
308 contents = "contents of file %s\n" % number
309 n = create_chk_filenode(contents, self.get_all_contents())
310 return contents, n, n.get_uri()
312 def makefile_mutable(self, number, mdmf=False):
313 contents = "contents of mutable file %s\n" % number
314 n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
315 return contents, n, n.get_uri(), n.get_readonly_uri()
318 return self.s.stopService()
320 def failUnlessIsBarDotTxt(self, res):
321 self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
323 def failUnlessIsQuuxDotTxt(self, res):
324 self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
326 def failUnlessIsBazDotTxt(self, res):
327 self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
329 def failUnlessIsSubBazDotTxt(self, res):
330 self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
332 def failUnlessIsBarJSON(self, res):
333 data = simplejson.loads(res)
334 self.failUnless(isinstance(data, list))
335 self.failUnlessEqual(data[0], "filenode")
336 self.failUnless(isinstance(data[1], dict))
337 self.failIf(data[1]["mutable"])
338 self.failIfIn("rw_uri", data[1]) # immutable
339 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
340 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
341 self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
343 def failUnlessIsQuuxJSON(self, res, readonly=False):
344 data = simplejson.loads(res)
345 self.failUnless(isinstance(data, list))
346 self.failUnlessEqual(data[0], "filenode")
347 self.failUnless(isinstance(data[1], dict))
349 return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
351 def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
352 self.failUnless(metadata['mutable'])
354 self.failIfIn("rw_uri", metadata)
356 self.failUnlessIn("rw_uri", metadata)
357 self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
358 self.failUnlessIn("ro_uri", metadata)
359 self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
360 self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
362 def failUnlessIsFooJSON(self, res):
363 data = simplejson.loads(res)
364 self.failUnless(isinstance(data, list))
365 self.failUnlessEqual(data[0], "dirnode", res)
366 self.failUnless(isinstance(data[1], dict))
367 self.failUnless(data[1]["mutable"])
368 self.failUnlessIn("rw_uri", data[1]) # mutable
369 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
370 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
371 self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
373 kidnames = sorted([unicode(n) for n in data[1]["children"]])
374 self.failUnlessEqual(kidnames,
375 [u"bar.txt", u"baz.txt", u"blockingfile",
376 u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
377 kids = dict( [(unicode(name),value)
379 in data[1]["children"].iteritems()] )
380 self.failUnlessEqual(kids[u"sub"][0], "dirnode")
381 self.failUnlessIn("metadata", kids[u"sub"][1])
382 self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
383 tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
384 self.failUnlessIn("linkcrtime", tahoe_md)
385 self.failUnlessIn("linkmotime", tahoe_md)
386 self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
387 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
388 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
389 self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
390 self._bar_txt_verifycap)
391 self.failUnlessIn("metadata", kids[u"bar.txt"][1])
392 self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
393 self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
394 self._bar_txt_metadata["tahoe"]["linkcrtime"])
395 self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
397 self.failUnlessIn("quux.txt", kids)
398 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
400 self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
401 self._quux_txt_readonly_uri)
403 def GET(self, urlpath, followRedirect=False, return_response=False,
405 # if return_response=True, this fires with (data, statuscode,
406 # respheaders) instead of just data.
407 assert not isinstance(urlpath, unicode)
408 url = self.webish_url + urlpath
409 factory = HTTPClientGETFactory(url, method="GET",
410 followRedirect=followRedirect, **kwargs)
411 reactor.connectTCP("localhost", self.webish_port, factory)
414 return (data, factory.status, factory.response_headers)
416 d.addCallback(_got_data)
417 return factory.deferred
419 def HEAD(self, urlpath, return_response=False, **kwargs):
420 # this requires some surgery, because twisted.web.client doesn't want
421 # to give us back the response headers.
422 factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
423 reactor.connectTCP("localhost", self.webish_port, factory)
426 return (data, factory.status, factory.response_headers)
428 d.addCallback(_got_data)
429 return factory.deferred
431 def PUT(self, urlpath, data, **kwargs):
432 url = self.webish_url + urlpath
433 return client.getPage(url, method="PUT", postdata=data, **kwargs)
435 def DELETE(self, urlpath):
436 url = self.webish_url + urlpath
437 return client.getPage(url, method="DELETE")
439 def POST(self, urlpath, followRedirect=False, **fields):
440 sepbase = "boogabooga"
444 form.append('Content-Disposition: form-data; name="_charset"')
448 for name, value in fields.iteritems():
449 if isinstance(value, tuple):
450 filename, value = value
451 form.append('Content-Disposition: form-data; name="%s"; '
452 'filename="%s"' % (name, filename.encode("utf-8")))
454 form.append('Content-Disposition: form-data; name="%s"' % name)
456 if isinstance(value, unicode):
457 value = value.encode("utf-8")
460 assert isinstance(value, str)
467 body = "\r\n".join(form) + "\r\n"
468 headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
469 return self.POST2(urlpath, body, headers, followRedirect)
471 def POST2(self, urlpath, body="", headers={}, followRedirect=False):
472 url = self.webish_url + urlpath
473 return client.getPage(url, method="POST", postdata=body,
474 headers=headers, followRedirect=followRedirect)
476 def shouldFail(self, res, expected_failure, which,
477 substring=None, response_substring=None):
478 if isinstance(res, failure.Failure):
479 res.trap(expected_failure)
481 self.failUnlessIn(substring, str(res), which)
482 if response_substring:
483 self.failUnlessIn(response_substring, res.value.response, which)
485 self.fail("%s was supposed to raise %s, not get '%s'" %
486 (which, expected_failure, res))
488 def shouldFail2(self, expected_failure, which, substring,
490 callable, *args, **kwargs):
491 assert substring is None or isinstance(substring, str)
492 assert response_substring is None or isinstance(response_substring, str)
493 d = defer.maybeDeferred(callable, *args, **kwargs)
495 if isinstance(res, failure.Failure):
496 res.trap(expected_failure)
498 self.failUnlessIn(substring, str(res),
499 "'%s' not in '%s' for test '%s'" % \
500 (substring, str(res), which))
501 if response_substring:
502 self.failUnlessIn(response_substring, res.value.response,
503 "'%s' not in '%s' for test '%s'" % \
504 (response_substring, res.value.response,
507 self.fail("%s was supposed to raise %s, not get '%s'" %
508 (which, expected_failure, res))
512 def should404(self, res, which):
513 if isinstance(res, failure.Failure):
514 res.trap(error.Error)
515 self.failUnlessReallyEqual(res.value.status, "404")
517 self.fail("%s was supposed to Error(404), not get '%s'" %
520 def should302(self, res, which):
521 if isinstance(res, failure.Failure):
522 res.trap(error.Error)
523 self.failUnlessReallyEqual(res.value.status, "302")
525 self.fail("%s was supposed to Error(302), not get '%s'" %
529 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
530 def test_create(self):
533 def test_welcome(self):
536 self.failUnlessIn('Welcome to Tahoe-LAFS', res)
537 self.failUnlessIn(FAVICON_MARKUP, res)
538 self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
540 self.s.basedir = 'web/test_welcome'
541 fileutil.make_dirs("web/test_welcome")
542 fileutil.make_dirs("web/test_welcome/private")
544 d.addCallback(_check)
547 def test_status(self):
548 h = self.s.get_history()
549 dl_num = h.list_all_download_statuses()[0].get_counter()
550 ul_num = h.list_all_upload_statuses()[0].get_counter()
551 mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
552 pub_num = h.list_all_publish_statuses()[0].get_counter()
553 ret_num = h.list_all_retrieve_statuses()[0].get_counter()
554 d = self.GET("/status", followRedirect=True)
556 self.failUnlessIn('Upload and Download Status', res)
557 self.failUnlessIn('"down-%d"' % dl_num, res)
558 self.failUnlessIn('"up-%d"' % ul_num, res)
559 self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
560 self.failUnlessIn('"publish-%d"' % pub_num, res)
561 self.failUnlessIn('"retrieve-%d"' % ret_num, res)
562 d.addCallback(_check)
563 d.addCallback(lambda res: self.GET("/status/?t=json"))
564 def _check_json(res):
565 data = simplejson.loads(res)
566 self.failUnless(isinstance(data, dict))
567 #active = data["active"]
568 # TODO: test more. We need a way to fake an active operation
570 d.addCallback(_check_json)
572 d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
574 self.failUnlessIn("File Download Status", res)
575 d.addCallback(_check_dl)
576 d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
577 def _check_dl_json(res):
578 data = simplejson.loads(res)
579 self.failUnless(isinstance(data, dict))
580 self.failUnlessIn("read", data)
581 self.failUnlessEqual(data["read"][0]["length"], 120)
582 self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
583 self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
584 self.failUnlessEqual(data["segment"][2]["finish_time"], None)
585 phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
586 cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
587 # serverids[] keys are strings, since that's what JSON does, but
588 # we'd really like them to be ints
589 self.failUnlessEqual(data["serverids"]["0"], "phwr")
590 self.failUnless(data["serverids"].has_key("1"), data["serverids"])
591 self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
592 self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
593 self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
594 self.failUnlessIn("dyhb", data)
595 self.failUnlessIn("misc", data)
596 d.addCallback(_check_dl_json)
597 d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
599 self.failUnlessIn("File Upload Status", res)
600 d.addCallback(_check_ul)
601 d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
602 def _check_mapupdate(res):
603 self.failUnlessIn("Mutable File Servermap Update Status", res)
604 d.addCallback(_check_mapupdate)
605 d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
606 def _check_publish(res):
607 self.failUnlessIn("Mutable File Publish Status", res)
608 d.addCallback(_check_publish)
609 d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
610 def _check_retrieve(res):
611 self.failUnlessIn("Mutable File Retrieve Status", res)
612 d.addCallback(_check_retrieve)
616 def test_status_numbers(self):
617 drrm = status.DownloadResultsRendererMixin()
618 self.failUnlessReallyEqual(drrm.render_time(None, None), "")
619 self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
620 self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
621 self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
622 self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
623 self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
624 self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
625 self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
626 self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
628 urrm = status.UploadResultsRendererMixin()
629 self.failUnlessReallyEqual(urrm.render_time(None, None), "")
630 self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
631 self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
632 self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
633 self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
634 self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
635 self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
636 self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
637 self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
639 def test_GET_FILEURL(self):
640 d = self.GET(self.public_url + "/foo/bar.txt")
641 d.addCallback(self.failUnlessIsBarDotTxt)
644 def test_GET_FILEURL_range(self):
645 headers = {"range": "bytes=1-10"}
646 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
647 return_response=True)
648 def _got((res, status, headers)):
649 self.failUnlessReallyEqual(int(status), 206)
650 self.failUnless(headers.has_key("content-range"))
651 self.failUnlessReallyEqual(headers["content-range"][0],
652 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
653 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
657 def test_GET_FILEURL_partial_range(self):
658 headers = {"range": "bytes=5-"}
659 length = len(self.BAR_CONTENTS)
660 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
661 return_response=True)
662 def _got((res, status, headers)):
663 self.failUnlessReallyEqual(int(status), 206)
664 self.failUnless(headers.has_key("content-range"))
665 self.failUnlessReallyEqual(headers["content-range"][0],
666 "bytes 5-%d/%d" % (length-1, length))
667 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
671 def test_GET_FILEURL_partial_end_range(self):
672 headers = {"range": "bytes=-5"}
673 length = len(self.BAR_CONTENTS)
674 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
675 return_response=True)
676 def _got((res, status, headers)):
677 self.failUnlessReallyEqual(int(status), 206)
678 self.failUnless(headers.has_key("content-range"))
679 self.failUnlessReallyEqual(headers["content-range"][0],
680 "bytes %d-%d/%d" % (length-5, length-1, length))
681 self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
685 def test_GET_FILEURL_partial_range_overrun(self):
686 headers = {"range": "bytes=100-200"}
687 d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
688 "416 Requested Range not satisfiable",
689 "First beyond end of file",
690 self.GET, self.public_url + "/foo/bar.txt",
694 def test_HEAD_FILEURL_range(self):
695 headers = {"range": "bytes=1-10"}
696 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
697 return_response=True)
698 def _got((res, status, headers)):
699 self.failUnlessReallyEqual(res, "")
700 self.failUnlessReallyEqual(int(status), 206)
701 self.failUnless(headers.has_key("content-range"))
702 self.failUnlessReallyEqual(headers["content-range"][0],
703 "bytes 1-10/%d" % len(self.BAR_CONTENTS))
707 def test_HEAD_FILEURL_partial_range(self):
708 headers = {"range": "bytes=5-"}
709 length = len(self.BAR_CONTENTS)
710 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
711 return_response=True)
712 def _got((res, status, headers)):
713 self.failUnlessReallyEqual(int(status), 206)
714 self.failUnless(headers.has_key("content-range"))
715 self.failUnlessReallyEqual(headers["content-range"][0],
716 "bytes 5-%d/%d" % (length-1, length))
720 def test_HEAD_FILEURL_partial_end_range(self):
721 headers = {"range": "bytes=-5"}
722 length = len(self.BAR_CONTENTS)
723 d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
724 return_response=True)
725 def _got((res, status, headers)):
726 self.failUnlessReallyEqual(int(status), 206)
727 self.failUnless(headers.has_key("content-range"))
728 self.failUnlessReallyEqual(headers["content-range"][0],
729 "bytes %d-%d/%d" % (length-5, length-1, length))
733 def test_HEAD_FILEURL_partial_range_overrun(self):
734 headers = {"range": "bytes=100-200"}
735 d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
736 "416 Requested Range not satisfiable",
738 self.HEAD, self.public_url + "/foo/bar.txt",
742 def test_GET_FILEURL_range_bad(self):
743 headers = {"range": "BOGUS=fizbop-quarnak"}
744 d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
745 return_response=True)
746 def _got((res, status, headers)):
747 self.failUnlessReallyEqual(int(status), 200)
748 self.failUnless(not headers.has_key("content-range"))
749 self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
753 def test_HEAD_FILEURL(self):
754 d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
755 def _got((res, status, headers)):
756 self.failUnlessReallyEqual(res, "")
757 self.failUnlessReallyEqual(headers["content-length"][0],
758 str(len(self.BAR_CONTENTS)))
759 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
763 def test_GET_FILEURL_named(self):
764 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
765 base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
766 d = self.GET(base + "/@@name=/blah.txt")
767 d.addCallback(self.failUnlessIsBarDotTxt)
768 d.addCallback(lambda res: self.GET(base + "/blah.txt"))
769 d.addCallback(self.failUnlessIsBarDotTxt)
770 d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
771 d.addCallback(self.failUnlessIsBarDotTxt)
772 d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
773 d.addCallback(self.failUnlessIsBarDotTxt)
774 save_url = base + "?save=true&filename=blah.txt"
775 d.addCallback(lambda res: self.GET(save_url))
776 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
777 u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
778 u_fn_e = urllib.quote(u_filename.encode("utf-8"))
779 u_url = base + "?save=true&filename=" + u_fn_e
780 d.addCallback(lambda res: self.GET(u_url))
781 d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
784 def test_PUT_FILEURL_named_bad(self):
785 base = "/file/%s" % urllib.quote(self._bar_txt_uri)
786 d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
788 "/file can only be used with GET or HEAD",
789 self.PUT, base + "/@@name=/blah.txt", "")
793 def test_GET_DIRURL_named_bad(self):
794 base = "/file/%s" % urllib.quote(self._foo_uri)
795 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
798 self.GET, base + "/@@name=/blah.txt")
801 def test_GET_slash_file_bad(self):
802 d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
804 "/file must be followed by a file-cap and a name",
808 def test_GET_unhandled_URI_named(self):
809 contents, n, newuri = self.makefile(12)
810 verifier_cap = n.get_verify_cap().to_string()
811 base = "/file/%s" % urllib.quote(verifier_cap)
812 # client.create_node_from_uri() can't handle verify-caps
813 d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
814 "400 Bad Request", "is not a file-cap",
818 def test_GET_unhandled_URI(self):
819 contents, n, newuri = self.makefile(12)
820 verifier_cap = n.get_verify_cap().to_string()
821 base = "/uri/%s" % urllib.quote(verifier_cap)
822 # client.create_node_from_uri() can't handle verify-caps
823 d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
825 "GET unknown URI type: can only do t=info",
829 def test_GET_FILE_URI(self):
830 base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
832 d.addCallback(self.failUnlessIsBarDotTxt)
835 def test_GET_FILE_URI_mdmf(self):
836 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
838 d.addCallback(self.failUnlessIsQuuxDotTxt)
841 def test_GET_FILE_URI_mdmf_extensions(self):
842 base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
844 d.addCallback(self.failUnlessIsQuuxDotTxt)
847 def test_GET_FILE_URI_mdmf_readonly(self):
848 base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
850 d.addCallback(self.failUnlessIsQuuxDotTxt)
853 def test_GET_FILE_URI_badchild(self):
854 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
855 errmsg = "Files have no children, certainly not named 'boguschild'"
856 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
857 "400 Bad Request", errmsg,
861 def test_PUT_FILE_URI_badchild(self):
862 base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
863 errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
864 d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
865 "400 Bad Request", errmsg,
869 def test_PUT_FILE_URI_mdmf(self):
870 base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
871 self._quux_new_contents = "new_contents"
873 d.addCallback(lambda res:
874 self.failUnlessIsQuuxDotTxt(res))
875 d.addCallback(lambda ignored:
876 self.PUT(base, self._quux_new_contents))
877 d.addCallback(lambda ignored:
879 d.addCallback(lambda res:
880 self.failUnlessReallyEqual(res, self._quux_new_contents))
883 def test_PUT_FILE_URI_mdmf_extensions(self):
884 base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
885 self._quux_new_contents = "new_contents"
887 d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
888 d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
889 d.addCallback(lambda ignored: self.GET(base))
890 d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
894 def test_PUT_FILE_URI_mdmf_readonly(self):
895 # We're not allowed to PUT things to a readonly cap.
896 base = "/uri/%s" % self._quux_txt_readonly_uri
898 d.addCallback(lambda res:
899 self.failUnlessIsQuuxDotTxt(res))
900 # What should we get here? We get a 500 error now; that's not right.
901 d.addCallback(lambda ignored:
902 self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
903 "400 Bad Request", "read-only cap",
904 self.PUT, base, "new data"))
907 def test_PUT_FILE_URI_sdmf_readonly(self):
908 # We're not allowed to put things to a readonly cap.
909 base = "/uri/%s" % self._baz_txt_readonly_uri
911 d.addCallback(lambda res:
912 self.failUnlessIsBazDotTxt(res))
913 d.addCallback(lambda ignored:
914 self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
915 "400 Bad Request", "read-only cap",
916 self.PUT, base, "new_data"))
919 def test_GET_etags(self):
921 def _check_etags(uri):
923 d2 = _get_etag(uri, 'json')
924 d = defer.DeferredList([d1, d2], consumeErrors=True)
926 # All deferred must succeed
927 self.failUnless(all([r[0] for r in results]))
928 # the etag for the t=json form should be just like the etag
929 # fo the default t='' form, but with a 'json' suffix
930 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
931 d.addCallback(_check)
934 def _get_etag(uri, t=''):
935 targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
936 d = self.GET(targetbase, return_response=True, followRedirect=True)
937 def _just_the_etag(result):
938 data, response, headers = result
939 etag = headers['etag'][0]
940 if uri.startswith('URI:DIR'):
941 self.failUnless(etag.startswith('DIR:'), etag)
943 return d.addCallback(_just_the_etag)
945 # Check that etags work with immutable directories
946 (newkids, caps) = self._create_immutable_children()
947 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
948 simplejson.dumps(newkids))
949 def _stash_immdir_uri(uri):
950 self._immdir_uri = uri
952 d.addCallback(_stash_immdir_uri)
953 d.addCallback(_check_etags)
955 # Check that etags work with immutable files
956 d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
958 # use the ETag on GET
959 def _check_match(ign):
960 uri = "/uri/%s" % self._bar_txt_uri
961 d = self.GET(uri, return_response=True)
963 d.addCallback(lambda (data, code, headers):
965 # do a GET that's supposed to match the ETag
966 d.addCallback(lambda etag:
967 self.GET(uri, return_response=True,
968 headers={"If-None-Match": etag}))
969 # make sure it short-circuited (304 instead of 200)
970 d.addCallback(lambda (data, code, headers):
971 self.failUnlessEqual(int(code), http.NOT_MODIFIED))
973 d.addCallback(_check_match)
975 def _no_etag(uri, t):
976 target = "/uri/%s?t=%s" % (uri, t)
977 d = self.GET(target, return_response=True, followRedirect=True)
978 d.addCallback(lambda (data, code, headers):
979 self.failIf("etag" in headers, target))
981 def _yes_etag(uri, t):
982 target = "/uri/%s?t=%s" % (uri, t)
983 d = self.GET(target, return_response=True, followRedirect=True)
984 d.addCallback(lambda (data, code, headers):
985 self.failUnless("etag" in headers, target))
988 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
989 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
990 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
991 d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
992 d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
994 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
995 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
996 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
997 d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
998 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
999 d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1003 # TODO: version of this with a Unicode filename
1004 def test_GET_FILEURL_save(self):
1005 d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1006 return_response=True)
1007 def _got((res, statuscode, headers)):
1008 content_disposition = headers["content-disposition"][0]
1009 self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1010 self.failUnlessIsBarDotTxt(res)
1014 def test_GET_FILEURL_missing(self):
1015 d = self.GET(self.public_url + "/foo/missing")
1016 d.addBoth(self.should404, "test_GET_FILEURL_missing")
1019 def test_GET_FILEURL_info_mdmf(self):
1020 d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1022 self.failUnlessIn("mutable file (mdmf)", res)
1023 self.failUnlessIn(self._quux_txt_uri, res)
1024 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1028 def test_GET_FILEURL_info_mdmf_readonly(self):
1029 d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1031 self.failUnlessIn("mutable file (mdmf)", res)
1032 self.failIfIn(self._quux_txt_uri, res)
1033 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1037 def test_GET_FILEURL_info_sdmf(self):
1038 d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1040 self.failUnlessIn("mutable file (sdmf)", res)
1041 self.failUnlessIn(self._baz_txt_uri, res)
1045 def test_GET_FILEURL_info_mdmf_extensions(self):
1046 d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1048 self.failUnlessIn("mutable file (mdmf)", res)
1049 self.failUnlessIn(self._quux_txt_uri, res)
1050 self.failUnlessIn(self._quux_txt_readonly_uri, res)
1054 def test_PUT_overwrite_only_files(self):
1055 # create a directory, put a file in that directory.
1056 contents, n, filecap = self.makefile(8)
1057 d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1058 d.addCallback(lambda res:
1059 self.PUT(self.public_url + "/foo/dir/file1.txt",
1060 self.NEWFILE_CONTENTS))
1061 # try to overwrite the file with replace=only-files
1062 # (this should work)
1063 d.addCallback(lambda res:
1064 self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1066 d.addCallback(lambda res:
1067 self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1068 "There was already a child by that name, and you asked me "
1069 "to not replace it",
1070 self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1074 def test_PUT_NEWFILEURL(self):
1075 d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1076 # TODO: we lose the response code, so we can't check this
1077 #self.failUnlessReallyEqual(responsecode, 201)
1078 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1079 d.addCallback(lambda res:
1080 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1081 self.NEWFILE_CONTENTS))
1084 def test_PUT_NEWFILEURL_not_mutable(self):
1085 d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1086 self.NEWFILE_CONTENTS)
1087 # TODO: we lose the response code, so we can't check this
1088 #self.failUnlessReallyEqual(responsecode, 201)
1089 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1090 d.addCallback(lambda res:
1091 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1092 self.NEWFILE_CONTENTS))
1095 def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1096 # this should get us a few segments of an MDMF mutable file,
1097 # which we can then test for.
1098 contents = self.NEWFILE_CONTENTS * 300000
1099 d = self.PUT("/uri?format=mdmf",
1101 def _got_filecap(filecap):
1102 self.failUnless(filecap.startswith("URI:MDMF"))
1104 d.addCallback(_got_filecap)
1105 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1106 d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1109 def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1110 contents = self.NEWFILE_CONTENTS * 300000
1111 d = self.PUT("/uri?format=sdmf",
1113 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1114 d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1117 def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1118 contents = self.NEWFILE_CONTENTS * 300000
1119 return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1120 400, "Bad Request", "Unknown format: foo",
1121 self.PUT, "/uri?format=foo",
1124 def test_PUT_NEWFILEURL_range_bad(self):
1125 headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1126 target = self.public_url + "/foo/new.txt"
1127 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1128 "501 Not Implemented",
1129 "Content-Range in PUT not yet supported",
1130 # (and certainly not for immutable files)
1131 self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1133 d.addCallback(lambda res:
1134 self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1137 def test_PUT_NEWFILEURL_mutable(self):
1138 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1139 self.NEWFILE_CONTENTS)
1140 # TODO: we lose the response code, so we can't check this
1141 #self.failUnlessReallyEqual(responsecode, 201)
1142 def _check_uri(res):
1143 u = uri.from_string_mutable_filenode(res)
1144 self.failUnless(u.is_mutable())
1145 self.failIf(u.is_readonly())
1147 d.addCallback(_check_uri)
1148 d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1149 d.addCallback(lambda res:
1150 self.failUnlessMutableChildContentsAre(self._foo_node,
1152 self.NEWFILE_CONTENTS))
1155 def test_PUT_NEWFILEURL_mutable_toobig(self):
1156 # It is okay to upload large mutable files, so we should be able
1158 d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1159 "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1162 def test_PUT_NEWFILEURL_replace(self):
1163 d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1164 # TODO: we lose the response code, so we can't check this
1165 #self.failUnlessReallyEqual(responsecode, 200)
1166 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1167 d.addCallback(lambda res:
1168 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1169 self.NEWFILE_CONTENTS))
1172 def test_PUT_NEWFILEURL_bad_t(self):
1173 d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1174 "PUT to a file: bad t=bogus",
1175 self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1179 def test_PUT_NEWFILEURL_no_replace(self):
1180 d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1181 self.NEWFILE_CONTENTS)
1182 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1184 "There was already a child by that name, and you asked me "
1185 "to not replace it")
1188 def test_PUT_NEWFILEURL_mkdirs(self):
1189 d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1191 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1192 d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1193 d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1194 d.addCallback(lambda res:
1195 self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1196 self.NEWFILE_CONTENTS))
1199 def test_PUT_NEWFILEURL_blocked(self):
1200 d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1201 self.NEWFILE_CONTENTS)
1202 d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1204 "Unable to create directory 'blockingfile': a file was in the way")
1207 def test_PUT_NEWFILEURL_emptyname(self):
1208 # an empty pathname component (i.e. a double-slash) is disallowed
1209 d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1211 "The webapi does not allow empty pathname components",
1212 self.PUT, self.public_url + "/foo//new.txt", "")
1215 def test_DELETE_FILEURL(self):
1216 d = self.DELETE(self.public_url + "/foo/bar.txt")
1217 d.addCallback(lambda res:
1218 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1221 def test_DELETE_FILEURL_missing(self):
1222 d = self.DELETE(self.public_url + "/foo/missing")
1223 d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1226 def test_DELETE_FILEURL_missing2(self):
1227 d = self.DELETE(self.public_url + "/missing/missing")
1228 d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1231 def failUnlessHasBarDotTxtMetadata(self, res):
1232 data = simplejson.loads(res)
1233 self.failUnless(isinstance(data, list))
1234 self.failUnlessIn("metadata", data[1])
1235 self.failUnlessIn("tahoe", data[1]["metadata"])
1236 self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1237 self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1238 self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1239 self._bar_txt_metadata["tahoe"]["linkcrtime"])
1241 def test_GET_FILEURL_json(self):
1242 # twisted.web.http.parse_qs ignores any query args without an '=', so
1243 # I can't do "GET /path?json", I have to do "GET /path/t=json"
1244 # instead. This may make it tricky to emulate the S3 interface
1246 d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1248 self.failUnlessIsBarJSON(data)
1249 self.failUnlessHasBarDotTxtMetadata(data)
1251 d.addCallback(_check1)
1254 def test_GET_FILEURL_json_mutable_type(self):
1255 # The JSON should include format, which says whether the
1256 # file is SDMF or MDMF
1257 d = self.PUT("/uri?format=mdmf",
1258 self.NEWFILE_CONTENTS * 300000)
1259 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1260 def _got_json(json, version):
1261 data = simplejson.loads(json)
1262 assert "filenode" == data[0]
1264 assert isinstance(data, dict)
1266 self.failUnlessIn("format", data)
1267 self.failUnlessEqual(data["format"], version)
1269 d.addCallback(_got_json, "MDMF")
1270 # Now make an SDMF file and check that it is reported correctly.
1271 d.addCallback(lambda ignored:
1272 self.PUT("/uri?format=sdmf",
1273 self.NEWFILE_CONTENTS * 300000))
1274 d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1275 d.addCallback(_got_json, "SDMF")
1278 def test_GET_FILEURL_json_mdmf(self):
1279 d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1280 d.addCallback(self.failUnlessIsQuuxJSON)
1283 def test_GET_FILEURL_json_missing(self):
1284 d = self.GET(self.public_url + "/foo/missing?json")
1285 d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1288 def test_GET_FILEURL_uri(self):
1289 d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1291 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1292 d.addCallback(_check)
1293 d.addCallback(lambda res:
1294 self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1296 # for now, for files, uris and readonly-uris are the same
1297 self.failUnlessReallyEqual(res, self._bar_txt_uri)
1298 d.addCallback(_check2)
1301 def test_GET_FILEURL_badtype(self):
1302 d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1305 self.public_url + "/foo/bar.txt?t=bogus")
1308 def test_CSS_FILE(self):
1309 d = self.GET("/tahoe.css", followRedirect=True)
1311 CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1312 self.failUnless(CSS_STYLE.search(res), res)
1313 d.addCallback(_check)
1316 def test_GET_FILEURL_uri_missing(self):
1317 d = self.GET(self.public_url + "/foo/missing?t=uri")
1318 d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1321 def _check_upload_and_mkdir_forms(self, html):
1322 # We should have a form to create a file, with radio buttons that allow
1323 # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1324 self.failUnlessIn('name="t" value="upload"', html)
1325 self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1326 self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1327 self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1329 # We should also have the ability to create a mutable directory, with
1330 # radio buttons that allow the user to toggle whether it is an SDMF (default)
1331 # or MDMF directory.
1332 self.failUnlessIn('name="t" value="mkdir"', html)
1333 self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1334 self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1336 self.failUnlessIn(FAVICON_MARKUP, html)
1338 def test_GET_DIRECTORY_html(self):
1339 d = self.GET(self.public_url + "/foo", followRedirect=True)
1341 self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1342 self._check_upload_and_mkdir_forms(html)
1343 self.failUnlessIn("quux", html)
1344 d.addCallback(_check)
1347 def test_GET_root_html(self):
1349 d.addCallback(self._check_upload_and_mkdir_forms)
1352 def test_GET_DIRURL(self):
1353 # the addSlash means we get a redirect here
1354 # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1356 d = self.GET(self.public_url + "/foo", followRedirect=True)
1358 self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1360 # the FILE reference points to a URI, but it should end in bar.txt
1361 bar_url = ("%s/file/%s/@@named=/bar.txt" %
1362 (ROOT, urllib.quote(self._bar_txt_uri)))
1363 get_bar = "".join([r'<td>FILE</td>',
1365 r'<a href="%s">bar.txt</a>' % bar_url,
1367 r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1369 self.failUnless(re.search(get_bar, res), res)
1370 for label in ['unlink', 'rename/move']:
1371 for line in res.split("\n"):
1372 # find the line that contains the relevant button for bar.txt
1373 if ("form action" in line and
1374 ('value="%s"' % (label,)) in line and
1375 'value="bar.txt"' in line):
1376 # the form target should use a relative URL
1377 foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1378 self.failUnlessIn('action="%s"' % foo_url, line)
1379 # and the when_done= should too
1380 #done_url = urllib.quote(???)
1381 #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1383 # 'unlink' needs to use POST because it directly has a side effect
1384 if label == 'unlink':
1385 self.failUnlessIn('method="post"', line)
1388 self.fail("unable to find '%s bar.txt' line" % (label,))
1390 # the DIR reference just points to a URI
1391 sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1392 get_sub = ((r'<td>DIR</td>')
1393 +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1394 self.failUnless(re.search(get_sub, res), res)
1395 d.addCallback(_check)
1397 # look at a readonly directory
1398 d.addCallback(lambda res:
1399 self.GET(self.public_url + "/reedownlee", followRedirect=True))
1401 self.failUnlessIn("(read-only)", res)
1402 self.failIfIn("Upload a file", res)
1403 d.addCallback(_check2)
1405 # and at a directory that contains a readonly directory
1406 d.addCallback(lambda res:
1407 self.GET(self.public_url, followRedirect=True))
1409 self.failUnless(re.search('<td>DIR-RO</td>'
1410 r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1411 d.addCallback(_check3)
1413 # and an empty directory
1414 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1416 self.failUnlessIn("directory is empty", res)
1417 MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1418 self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1419 d.addCallback(_check4)
1421 # and at a literal directory
1422 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1423 d.addCallback(lambda res:
1424 self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1426 self.failUnlessIn('(immutable)', res)
1427 self.failUnless(re.search('<td>FILE</td>'
1428 r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1429 d.addCallback(_check5)
1432 def test_GET_DIRURL_badtype(self):
1433 d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1437 self.public_url + "/foo?t=bogus")
1440 def test_GET_DIRURL_json(self):
1441 d = self.GET(self.public_url + "/foo?t=json")
1442 d.addCallback(self.failUnlessIsFooJSON)
1445 def test_GET_DIRURL_json_format(self):
1446 d = self.PUT(self.public_url + \
1447 "/foo/sdmf.txt?format=sdmf",
1448 self.NEWFILE_CONTENTS * 300000)
1449 d.addCallback(lambda ignored:
1450 self.PUT(self.public_url + \
1451 "/foo/mdmf.txt?format=mdmf",
1452 self.NEWFILE_CONTENTS * 300000))
1453 # Now we have an MDMF and SDMF file in the directory. If we GET
1454 # its JSON, we should see their encodings.
1455 d.addCallback(lambda ignored:
1456 self.GET(self.public_url + "/foo?t=json"))
1457 def _got_json(json):
1458 data = simplejson.loads(json)
1459 assert data[0] == "dirnode"
1462 kids = data['children']
1464 mdmf_data = kids['mdmf.txt'][1]
1465 self.failUnlessIn("format", mdmf_data)
1466 self.failUnlessEqual(mdmf_data["format"], "MDMF")
1468 sdmf_data = kids['sdmf.txt'][1]
1469 self.failUnlessIn("format", sdmf_data)
1470 self.failUnlessEqual(sdmf_data["format"], "SDMF")
1471 d.addCallback(_got_json)
1475 def test_POST_DIRURL_manifest_no_ophandle(self):
1476 d = self.shouldFail2(error.Error,
1477 "test_POST_DIRURL_manifest_no_ophandle",
1479 "slow operation requires ophandle=",
1480 self.POST, self.public_url, t="start-manifest")
1483 def test_POST_DIRURL_manifest(self):
1484 d = defer.succeed(None)
1485 def getman(ignored, output):
1486 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1487 followRedirect=True)
1488 d.addCallback(self.wait_for_operation, "125")
1489 d.addCallback(self.get_operation_results, "125", output)
1491 d.addCallback(getman, None)
1492 def _got_html(manifest):
1493 self.failUnlessIn("Manifest of SI=", manifest)
1494 self.failUnlessIn("<td>sub</td>", manifest)
1495 self.failUnlessIn(self._sub_uri, manifest)
1496 self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1497 self.failUnlessIn(FAVICON_MARKUP, manifest)
1498 d.addCallback(_got_html)
1500 # both t=status and unadorned GET should be identical
1501 d.addCallback(lambda res: self.GET("/operations/125"))
1502 d.addCallback(_got_html)
1504 d.addCallback(getman, "html")
1505 d.addCallback(_got_html)
1506 d.addCallback(getman, "text")
1507 def _got_text(manifest):
1508 self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1509 self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1510 d.addCallback(_got_text)
1511 d.addCallback(getman, "JSON")
1513 data = res["manifest"]
1515 for (path_list, cap) in data:
1516 got[tuple(path_list)] = cap
1517 self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1518 self.failUnlessIn((u"sub", u"baz.txt"), got)
1519 self.failUnlessIn("finished", res)
1520 self.failUnlessIn("origin", res)
1521 self.failUnlessIn("storage-index", res)
1522 self.failUnlessIn("verifycaps", res)
1523 self.failUnlessIn("stats", res)
1524 d.addCallback(_got_json)
1527 def test_POST_DIRURL_deepsize_no_ophandle(self):
1528 d = self.shouldFail2(error.Error,
1529 "test_POST_DIRURL_deepsize_no_ophandle",
1531 "slow operation requires ophandle=",
1532 self.POST, self.public_url, t="start-deep-size")
1535 def test_POST_DIRURL_deepsize(self):
1536 d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1537 followRedirect=True)
1538 d.addCallback(self.wait_for_operation, "126")
1539 d.addCallback(self.get_operation_results, "126", "json")
1540 def _got_json(data):
1541 self.failUnlessReallyEqual(data["finished"], True)
1543 self.failUnless(size > 1000)
1544 d.addCallback(_got_json)
1545 d.addCallback(self.get_operation_results, "126", "text")
1547 mo = re.search(r'^size: (\d+)$', res, re.M)
1548 self.failUnless(mo, res)
1549 size = int(mo.group(1))
1550 # with directories, the size varies.
1551 self.failUnless(size > 1000)
1552 d.addCallback(_got_text)
1555 def test_POST_DIRURL_deepstats_no_ophandle(self):
1556 d = self.shouldFail2(error.Error,
1557 "test_POST_DIRURL_deepstats_no_ophandle",
1559 "slow operation requires ophandle=",
1560 self.POST, self.public_url, t="start-deep-stats")
1563 def test_POST_DIRURL_deepstats(self):
1564 d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1565 followRedirect=True)
1566 d.addCallback(self.wait_for_operation, "127")
1567 d.addCallback(self.get_operation_results, "127", "json")
1568 def _got_json(stats):
1569 expected = {"count-immutable-files": 3,
1570 "count-mutable-files": 2,
1571 "count-literal-files": 0,
1573 "count-directories": 3,
1574 "size-immutable-files": 57,
1575 "size-literal-files": 0,
1576 #"size-directories": 1912, # varies
1577 #"largest-directory": 1590,
1578 "largest-directory-children": 7,
1579 "largest-immutable-file": 19,
1581 for k,v in expected.iteritems():
1582 self.failUnlessReallyEqual(stats[k], v,
1583 "stats[%s] was %s, not %s" %
1585 self.failUnlessReallyEqual(stats["size-files-histogram"],
1587 d.addCallback(_got_json)
1590 def test_POST_DIRURL_stream_manifest(self):
1591 d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1593 self.failUnless(res.endswith("\n"))
1594 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1595 self.failUnlessReallyEqual(len(units), 9)
1596 self.failUnlessEqual(units[-1]["type"], "stats")
1598 self.failUnlessEqual(first["path"], [])
1599 self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1600 self.failUnlessEqual(first["type"], "directory")
1601 baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1602 self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1603 self.failIfEqual(baz["storage-index"], None)
1604 self.failIfEqual(baz["verifycap"], None)
1605 self.failIfEqual(baz["repaircap"], None)
1606 # XXX: Add quux and baz to this test.
1608 d.addCallback(_check)
1611 def test_GET_DIRURL_uri(self):
1612 d = self.GET(self.public_url + "/foo?t=uri")
1614 self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1615 d.addCallback(_check)
1618 def test_GET_DIRURL_readonly_uri(self):
1619 d = self.GET(self.public_url + "/foo?t=readonly-uri")
1621 self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1622 d.addCallback(_check)
1625 def test_PUT_NEWDIRURL(self):
1626 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1627 d.addCallback(lambda res:
1628 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630 d.addCallback(self.failUnlessNodeKeysAre, [])
1633 def test_PUT_NEWDIRURL_mdmf(self):
1634 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1635 d.addCallback(lambda res:
1636 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638 d.addCallback(lambda node:
1639 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1642 def test_PUT_NEWDIRURL_sdmf(self):
1643 d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1645 d.addCallback(lambda res:
1646 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648 d.addCallback(lambda node:
1649 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1652 def test_PUT_NEWDIRURL_bad_format(self):
1653 return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1654 400, "Bad Request", "Unknown format: foo",
1655 self.PUT, self.public_url +
1656 "/foo/newdir=?t=mkdir&format=foo", "")
1658 def test_POST_NEWDIRURL(self):
1659 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1660 d.addCallback(lambda res:
1661 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1662 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1663 d.addCallback(self.failUnlessNodeKeysAre, [])
1666 def test_POST_NEWDIRURL_mdmf(self):
1667 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1668 d.addCallback(lambda res:
1669 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1670 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1671 d.addCallback(lambda node:
1672 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1675 def test_POST_NEWDIRURL_sdmf(self):
1676 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1677 d.addCallback(lambda res:
1678 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1679 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680 d.addCallback(lambda node:
1681 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1684 def test_POST_NEWDIRURL_bad_format(self):
1685 return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1686 400, "Bad Request", "Unknown format: foo",
1687 self.POST2, self.public_url + \
1688 "/foo/newdir?t=mkdir&format=foo", "")
1690 def test_POST_NEWDIRURL_emptyname(self):
1691 # an empty pathname component (i.e. a double-slash) is disallowed
1692 d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1694 "The webapi does not allow empty pathname components, i.e. a double slash",
1695 self.POST, self.public_url + "//?t=mkdir")
1698 def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1699 (newkids, caps) = self._create_initial_children()
1700 query = "/foo/newdir?t=mkdir-with-children"
1701 if version == MDMF_VERSION:
1702 query += "&format=mdmf"
1703 elif version == SDMF_VERSION:
1704 query += "&format=sdmf"
1706 version = SDMF_VERSION # for later
1707 d = self.POST2(self.public_url + query,
1708 simplejson.dumps(newkids))
1710 n = self.s.create_node_from_uri(uri.strip())
1711 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1712 self.failUnlessEqual(n._node.get_version(), version)
1713 d2.addCallback(lambda ign:
1714 self.failUnlessROChildURIIs(n, u"child-imm",
1716 d2.addCallback(lambda ign:
1717 self.failUnlessRWChildURIIs(n, u"child-mutable",
1719 d2.addCallback(lambda ign:
1720 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1722 d2.addCallback(lambda ign:
1723 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1724 caps['unknown_rocap']))
1725 d2.addCallback(lambda ign:
1726 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1727 caps['unknown_rwcap']))
1728 d2.addCallback(lambda ign:
1729 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1730 caps['unknown_immcap']))
1731 d2.addCallback(lambda ign:
1732 self.failUnlessRWChildURIIs(n, u"dirchild",
1734 d2.addCallback(lambda ign:
1735 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1737 d2.addCallback(lambda ign:
1738 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1739 caps['emptydircap']))
1741 d.addCallback(_check)
1742 d.addCallback(lambda res:
1743 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1746 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1750 def test_POST_NEWDIRURL_initial_children(self):
1751 return self._do_POST_NEWDIRURL_initial_children_test()
1753 def test_POST_NEWDIRURL_initial_children_mdmf(self):
1754 return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1756 def test_POST_NEWDIRURL_initial_children_sdmf(self):
1757 return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1759 def test_POST_NEWDIRURL_initial_children_bad_format(self):
1760 (newkids, caps) = self._create_initial_children()
1761 return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1762 400, "Bad Request", "Unknown format: foo",
1763 self.POST2, self.public_url + \
1764 "/foo/newdir?t=mkdir-with-children&format=foo",
1765 simplejson.dumps(newkids))
1767 def test_POST_NEWDIRURL_immutable(self):
1768 (newkids, caps) = self._create_immutable_children()
1769 d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1770 simplejson.dumps(newkids))
1772 n = self.s.create_node_from_uri(uri.strip())
1773 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1774 d2.addCallback(lambda ign:
1775 self.failUnlessROChildURIIs(n, u"child-imm",
1777 d2.addCallback(lambda ign:
1778 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1779 caps['unknown_immcap']))
1780 d2.addCallback(lambda ign:
1781 self.failUnlessROChildURIIs(n, u"dirchild-imm",
1783 d2.addCallback(lambda ign:
1784 self.failUnlessROChildURIIs(n, u"dirchild-lit",
1786 d2.addCallback(lambda ign:
1787 self.failUnlessROChildURIIs(n, u"dirchild-empty",
1788 caps['emptydircap']))
1790 d.addCallback(_check)
1791 d.addCallback(lambda res:
1792 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1795 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1796 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1797 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1799 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1800 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1801 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1802 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1803 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1805 d.addErrback(self.explain_web_error)
1808 def test_POST_NEWDIRURL_immutable_bad(self):
1809 (newkids, caps) = self._create_initial_children()
1810 d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1812 "needed to be immutable but was not",
1814 self.public_url + "/foo/newdir?t=mkdir-immutable",
1815 simplejson.dumps(newkids))
1818 def test_PUT_NEWDIRURL_exists(self):
1819 d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1820 d.addCallback(lambda res:
1821 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1822 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1823 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1826 def test_PUT_NEWDIRURL_blocked(self):
1827 d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1828 "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1830 self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1831 d.addCallback(lambda res:
1832 self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1833 d.addCallback(lambda res: self._foo_node.get(u"sub"))
1834 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1837 def test_PUT_NEWDIRURL_mkdirs(self):
1838 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1839 d.addCallback(lambda res:
1840 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1841 d.addCallback(lambda res:
1842 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1843 d.addCallback(lambda res:
1844 self._foo_node.get_child_at_path(u"subdir/newdir"))
1845 d.addCallback(self.failUnlessNodeKeysAre, [])
1848 def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1849 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1850 d.addCallback(lambda ignored:
1851 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1852 d.addCallback(lambda ignored:
1853 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1854 d.addCallback(lambda ignored:
1855 self._foo_node.get_child_at_path(u"subdir"))
1856 def _got_subdir(subdir):
1857 # XXX: What we want?
1858 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1859 self.failUnlessNodeHasChild(subdir, u"newdir")
1860 return subdir.get_child_at_path(u"newdir")
1861 d.addCallback(_got_subdir)
1862 d.addCallback(lambda newdir:
1863 self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1866 def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1867 d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1868 d.addCallback(lambda ignored:
1869 self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1870 d.addCallback(lambda ignored:
1871 self.failIfNodeHasChild(self._foo_node, u"newdir"))
1872 d.addCallback(lambda ignored:
1873 self._foo_node.get_child_at_path(u"subdir"))
1874 def _got_subdir(subdir):
1875 # XXX: What we want?
1876 #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1877 self.failUnlessNodeHasChild(subdir, u"newdir")
1878 return subdir.get_child_at_path(u"newdir")
1879 d.addCallback(_got_subdir)
1880 d.addCallback(lambda newdir:
1881 self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1884 def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1885 return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1886 400, "Bad Request", "Unknown format: foo",
1887 self.PUT, self.public_url + \
1888 "/foo/subdir/newdir?t=mkdir&format=foo",
1891 def test_DELETE_DIRURL(self):
1892 d = self.DELETE(self.public_url + "/foo")
1893 d.addCallback(lambda res:
1894 self.failIfNodeHasChild(self.public_root, u"foo"))
1897 def test_DELETE_DIRURL_missing(self):
1898 d = self.DELETE(self.public_url + "/foo/missing")
1899 d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1900 d.addCallback(lambda res:
1901 self.failUnlessNodeHasChild(self.public_root, u"foo"))
1904 def test_DELETE_DIRURL_missing2(self):
1905 d = self.DELETE(self.public_url + "/missing")
1906 d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1909 def dump_root(self):
1911 w = webish.DirnodeWalkerMixin()
1912 def visitor(childpath, childnode, metadata):
1914 d = w.walk(self.public_root, visitor)
1917 def failUnlessNodeKeysAre(self, node, expected_keys):
1918 for k in expected_keys:
1919 assert isinstance(k, unicode)
1921 def _check(children):
1922 self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1923 d.addCallback(_check)
1925 def failUnlessNodeHasChild(self, node, name):
1926 assert isinstance(name, unicode)
1928 def _check(children):
1929 self.failUnlessIn(name, children)
1930 d.addCallback(_check)
1932 def failIfNodeHasChild(self, node, name):
1933 assert isinstance(name, unicode)
1935 def _check(children):
1936 self.failIfIn(name, children)
1937 d.addCallback(_check)
1940 def failUnlessChildContentsAre(self, node, name, expected_contents):
1941 assert isinstance(name, unicode)
1942 d = node.get_child_at_path(name)
1943 d.addCallback(lambda node: download_to_data(node))
1944 def _check(contents):
1945 self.failUnlessReallyEqual(contents, expected_contents)
1946 d.addCallback(_check)
1949 def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1950 assert isinstance(name, unicode)
1951 d = node.get_child_at_path(name)
1952 d.addCallback(lambda node: node.download_best_version())
1953 def _check(contents):
1954 self.failUnlessReallyEqual(contents, expected_contents)
1955 d.addCallback(_check)
1958 def failUnlessRWChildURIIs(self, node, name, expected_uri):
1959 assert isinstance(name, unicode)
1960 d = node.get_child_at_path(name)
1962 self.failUnless(child.is_unknown() or not child.is_readonly())
1963 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1964 self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1965 expected_ro_uri = self._make_readonly(expected_uri)
1967 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1968 d.addCallback(_check)
1971 def failUnlessROChildURIIs(self, node, name, expected_uri):
1972 assert isinstance(name, unicode)
1973 d = node.get_child_at_path(name)
1975 self.failUnless(child.is_unknown() or child.is_readonly())
1976 self.failUnlessReallyEqual(child.get_write_uri(), None)
1977 self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1978 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1979 d.addCallback(_check)
1982 def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1983 assert isinstance(name, unicode)
1984 d = node.get_child_at_path(name)
1986 self.failUnless(child.is_unknown() or not child.is_readonly())
1987 self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1988 self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1989 expected_ro_uri = self._make_readonly(got_uri)
1991 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1992 d.addCallback(_check)
1995 def failUnlessURIMatchesROChild(self, got_uri, node, name):
1996 assert isinstance(name, unicode)
1997 d = node.get_child_at_path(name)
1999 self.failUnless(child.is_unknown() or child.is_readonly())
2000 self.failUnlessReallyEqual(child.get_write_uri(), None)
2001 self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2002 self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2003 d.addCallback(_check)
2006 def failUnlessCHKURIHasContents(self, got_uri, contents):
2007 self.failUnless(self.get_all_contents()[got_uri] == contents)
2009 def test_POST_upload(self):
2010 d = self.POST(self.public_url + "/foo", t="upload",
2011 file=("new.txt", self.NEWFILE_CONTENTS))
2013 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2014 d.addCallback(lambda res:
2015 self.failUnlessChildContentsAre(fn, u"new.txt",
2016 self.NEWFILE_CONTENTS))
2019 def test_POST_upload_unicode(self):
2020 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2021 d = self.POST(self.public_url + "/foo", t="upload",
2022 file=(filename, self.NEWFILE_CONTENTS))
2024 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2025 d.addCallback(lambda res:
2026 self.failUnlessChildContentsAre(fn, filename,
2027 self.NEWFILE_CONTENTS))
2028 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2029 d.addCallback(lambda res: self.GET(target_url))
2030 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2031 self.NEWFILE_CONTENTS,
2035 def test_POST_upload_unicode_named(self):
2036 filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2037 d = self.POST(self.public_url + "/foo", t="upload",
2039 file=("overridden", self.NEWFILE_CONTENTS))
2041 d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2042 d.addCallback(lambda res:
2043 self.failUnlessChildContentsAre(fn, filename,
2044 self.NEWFILE_CONTENTS))
2045 target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2046 d.addCallback(lambda res: self.GET(target_url))
2047 d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2048 self.NEWFILE_CONTENTS,
2052 def test_POST_upload_no_link(self):
2053 d = self.POST("/uri", t="upload",
2054 file=("new.txt", self.NEWFILE_CONTENTS))
2055 def _check_upload_results(page):
2056 # this should be a page which describes the results of the upload
2057 # that just finished.
2058 self.failUnlessIn("Upload Results:", page)
2059 self.failUnlessIn("URI:", page)
2060 uri_re = re.compile("URI: <tt><span>(.*)</span>")
2061 mo = uri_re.search(page)
2062 self.failUnless(mo, page)
2063 new_uri = mo.group(1)
2065 d.addCallback(_check_upload_results)
2066 d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2069 def test_POST_upload_no_link_whendone(self):
2070 d = self.POST("/uri", t="upload", when_done="/",
2071 file=("new.txt", self.NEWFILE_CONTENTS))
2072 d.addBoth(self.shouldRedirect, "/")
2075 def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2076 d = defer.maybeDeferred(callable, *args, **kwargs)
2078 if isinstance(res, failure.Failure):
2079 res.trap(error.PageRedirect)
2080 statuscode = res.value.status
2081 target = res.value.location
2082 return checker(statuscode, target)
2083 self.fail("%s: callable was supposed to redirect, not return '%s'"
2088 def test_POST_upload_no_link_whendone_results(self):
2089 def check(statuscode, target):
2090 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2091 self.failUnless(target.startswith(self.webish_url), target)
2092 return client.getPage(target, method="GET")
2093 d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2095 self.POST, "/uri", t="upload",
2096 when_done="/uri/%(uri)s",
2097 file=("new.txt", self.NEWFILE_CONTENTS))
2098 d.addCallback(lambda res:
2099 self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2102 def test_POST_upload_no_link_mutable(self):
2103 d = self.POST("/uri", t="upload", mutable="true",
2104 file=("new.txt", self.NEWFILE_CONTENTS))
2105 def _check(filecap):
2106 filecap = filecap.strip()
2107 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2108 self.filecap = filecap
2109 u = uri.WriteableSSKFileURI.init_from_string(filecap)
2110 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2111 n = self.s.create_node_from_uri(filecap)
2112 return n.download_best_version()
2113 d.addCallback(_check)
2115 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2116 return self.GET("/uri/%s" % urllib.quote(self.filecap))
2117 d.addCallback(_check2)
2119 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2120 return self.GET("/file/%s" % urllib.quote(self.filecap))
2121 d.addCallback(_check3)
2123 self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2124 d.addCallback(_check4)
2127 def test_POST_upload_no_link_mutable_toobig(self):
2128 # The SDMF size limit is no longer in place, so we should be
2129 # able to upload mutable files that are as large as we want them
2131 d = self.POST("/uri", t="upload", mutable="true",
2132 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2136 def test_POST_upload_format_unlinked(self):
2137 def _check_upload_unlinked(ign, format, uri_prefix):
2138 filename = format + ".txt"
2139 d = self.POST("/uri?t=upload&format=" + format,
2140 file=(filename, self.NEWFILE_CONTENTS * 300000))
2141 def _got_results(results):
2142 if format.upper() in ("SDMF", "MDMF"):
2143 # webapi.rst says this returns a filecap
2146 # for immutable, it returns an "upload results page", and
2147 # the filecap is buried inside
2148 line = [l for l in results.split("\n") if "URI: " in l][0]
2149 mo = re.search(r'<span>([^<]+)</span>', line)
2150 filecap = mo.group(1)
2151 self.failUnless(filecap.startswith(uri_prefix),
2152 (uri_prefix, filecap))
2153 return self.GET("/uri/%s?t=json" % filecap)
2154 d.addCallback(_got_results)
2155 def _got_json(json):
2156 data = simplejson.loads(json)
2158 self.failUnlessIn("format", data)
2159 self.failUnlessEqual(data["format"], format.upper())
2160 d.addCallback(_got_json)
2162 d = defer.succeed(None)
2163 d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2164 d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2165 d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2166 d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2169 def test_POST_upload_bad_format_unlinked(self):
2170 return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2171 400, "Bad Request", "Unknown format: foo",
2173 "/uri?t=upload&format=foo",
2174 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2176 def test_POST_upload_format(self):
2177 def _check_upload(ign, format, uri_prefix, fn=None):
2178 filename = format + ".txt"
2179 d = self.POST(self.public_url +
2180 "/foo?t=upload&format=" + format,
2181 file=(filename, self.NEWFILE_CONTENTS * 300000))
2182 def _got_filecap(filecap):
2184 filenameu = unicode(filename)
2185 self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2186 self.failUnless(filecap.startswith(uri_prefix))
2187 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2188 d.addCallback(_got_filecap)
2189 def _got_json(json):
2190 data = simplejson.loads(json)
2192 self.failUnlessIn("format", data)
2193 self.failUnlessEqual(data["format"], format.upper())
2194 d.addCallback(_got_json)
2197 d = defer.succeed(None)
2198 d.addCallback(_check_upload, "chk", "URI:CHK")
2199 d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2200 d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2201 d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2204 def test_POST_upload_bad_format(self):
2205 return self.shouldHTTPError("POST_upload_bad_format",
2206 400, "Bad Request", "Unknown format: foo",
2207 self.POST, self.public_url + \
2208 "/foo?t=upload&format=foo",
2209 file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2211 def test_POST_upload_mutable(self):
2212 # this creates a mutable file
2213 d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2214 file=("new.txt", self.NEWFILE_CONTENTS))
2216 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2217 d.addCallback(lambda res:
2218 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2219 self.NEWFILE_CONTENTS))
2220 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2222 self.failUnless(IMutableFileNode.providedBy(newnode))
2223 self.failUnless(newnode.is_mutable())
2224 self.failIf(newnode.is_readonly())
2225 self._mutable_node = newnode
2226 self._mutable_uri = newnode.get_uri()
2229 # now upload it again and make sure that the URI doesn't change
2230 NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2231 d.addCallback(lambda res:
2232 self.POST(self.public_url + "/foo", t="upload",
2234 file=("new.txt", NEWER_CONTENTS)))
2235 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2236 d.addCallback(lambda res:
2237 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2239 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2241 self.failUnless(IMutableFileNode.providedBy(newnode))
2242 self.failUnless(newnode.is_mutable())
2243 self.failIf(newnode.is_readonly())
2244 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2245 d.addCallback(_got2)
2247 # upload a second time, using PUT instead of POST
2248 NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2249 d.addCallback(lambda res:
2250 self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2251 d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2252 d.addCallback(lambda res:
2253 self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2256 # finally list the directory, since mutable files are displayed
2257 # slightly differently
2259 d.addCallback(lambda res:
2260 self.GET(self.public_url + "/foo/",
2261 followRedirect=True))
2262 def _check_page(res):
2263 # TODO: assert more about the contents
2264 self.failUnlessIn("SSK", res)
2266 d.addCallback(_check_page)
2268 d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2270 self.failUnless(IMutableFileNode.providedBy(newnode))
2271 self.failUnless(newnode.is_mutable())
2272 self.failIf(newnode.is_readonly())
2273 self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2274 d.addCallback(_got3)
2276 # look at the JSON form of the enclosing directory
2277 d.addCallback(lambda res:
2278 self.GET(self.public_url + "/foo/?t=json",
2279 followRedirect=True))
2280 def _check_page_json(res):
2281 parsed = simplejson.loads(res)
2282 self.failUnlessEqual(parsed[0], "dirnode")
2283 children = dict( [(unicode(name),value)
2285 in parsed[1]["children"].iteritems()] )
2286 self.failUnlessIn(u"new.txt", children)
2287 new_json = children[u"new.txt"]
2288 self.failUnlessEqual(new_json[0], "filenode")
2289 self.failUnless(new_json[1]["mutable"])
2290 self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2291 ro_uri = self._mutable_node.get_readonly().to_string()
2292 self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2293 d.addCallback(_check_page_json)
2295 # and the JSON form of the file
2296 d.addCallback(lambda res:
2297 self.GET(self.public_url + "/foo/new.txt?t=json"))
2298 def _check_file_json(res):
2299 parsed = simplejson.loads(res)
2300 self.failUnlessEqual(parsed[0], "filenode")
2301 self.failUnless(parsed[1]["mutable"])
2302 self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2303 ro_uri = self._mutable_node.get_readonly().to_string()
2304 self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2305 d.addCallback(_check_file_json)
2307 # and look at t=uri and t=readonly-uri
2308 d.addCallback(lambda res:
2309 self.GET(self.public_url + "/foo/new.txt?t=uri"))
2310 d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2311 d.addCallback(lambda res:
2312 self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2313 def _check_ro_uri(res):
2314 ro_uri = self._mutable_node.get_readonly().to_string()
2315 self.failUnlessReallyEqual(res, ro_uri)
2316 d.addCallback(_check_ro_uri)
2318 # make sure we can get to it from /uri/URI
2319 d.addCallback(lambda res:
2320 self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2321 d.addCallback(lambda res:
2322 self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2324 # and that HEAD computes the size correctly
2325 d.addCallback(lambda res:
2326 self.HEAD(self.public_url + "/foo/new.txt",
2327 return_response=True))
2328 def _got_headers((res, status, headers)):
2329 self.failUnlessReallyEqual(res, "")
2330 self.failUnlessReallyEqual(headers["content-length"][0],
2331 str(len(NEW2_CONTENTS)))
2332 self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2333 d.addCallback(_got_headers)
2335 # make sure that outdated size limits aren't enforced anymore.
2336 d.addCallback(lambda ignored:
2337 self.POST(self.public_url + "/foo", t="upload",
2340 "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2341 d.addErrback(self.dump_error)
2344 def test_POST_upload_mutable_toobig(self):
2345 # SDMF had a size limti that was removed a while ago. MDMF has
2346 # never had a size limit. Test to make sure that we do not
2347 # encounter errors when trying to upload large mutable files,
2348 # since there should be no coded prohibitions regarding large
2350 d = self.POST(self.public_url + "/foo",
2351 t="upload", mutable="true",
2352 file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2355 def dump_error(self, f):
2356 # if the web server returns an error code (like 400 Bad Request),
2357 # web.client.getPage puts the HTTP response body into the .response
2358 # attribute of the exception object that it gives back. It does not
2359 # appear in the Failure's repr(), so the ERROR that trial displays
2360 # will be rather terse and unhelpful. addErrback this method to the
2361 # end of your chain to get more information out of these errors.
2362 if f.check(error.Error):
2363 print "web.error.Error:"
2365 print f.value.response
2368 def test_POST_upload_replace(self):
2369 d = self.POST(self.public_url + "/foo", t="upload",
2370 file=("bar.txt", self.NEWFILE_CONTENTS))
2372 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2373 d.addCallback(lambda res:
2374 self.failUnlessChildContentsAre(fn, u"bar.txt",
2375 self.NEWFILE_CONTENTS))
2378 def test_POST_upload_no_replace_ok(self):
2379 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2380 file=("new.txt", self.NEWFILE_CONTENTS))
2381 d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2382 d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2383 self.NEWFILE_CONTENTS))
2386 def test_POST_upload_no_replace_queryarg(self):
2387 d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2388 file=("bar.txt", self.NEWFILE_CONTENTS))
2389 d.addBoth(self.shouldFail, error.Error,
2390 "POST_upload_no_replace_queryarg",
2392 "There was already a child by that name, and you asked me "
2393 "to not replace it")
2394 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2395 d.addCallback(self.failUnlessIsBarDotTxt)
2398 def test_POST_upload_no_replace_field(self):
2399 d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2400 file=("bar.txt", self.NEWFILE_CONTENTS))
2401 d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2403 "There was already a child by that name, and you asked me "
2404 "to not replace it")
2405 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2406 d.addCallback(self.failUnlessIsBarDotTxt)
2409 def test_POST_upload_whendone(self):
2410 d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2411 file=("new.txt", self.NEWFILE_CONTENTS))
2412 d.addBoth(self.shouldRedirect, "/THERE")
2414 d.addCallback(lambda res:
2415 self.failUnlessChildContentsAre(fn, u"new.txt",
2416 self.NEWFILE_CONTENTS))
2419 def test_POST_upload_named(self):
2421 d = self.POST(self.public_url + "/foo", t="upload",
2422 name="new.txt", file=self.NEWFILE_CONTENTS)
2423 d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2424 d.addCallback(lambda res:
2425 self.failUnlessChildContentsAre(fn, u"new.txt",
2426 self.NEWFILE_CONTENTS))
2429 def test_POST_upload_named_badfilename(self):
2430 d = self.POST(self.public_url + "/foo", t="upload",
2431 name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2432 d.addBoth(self.shouldFail, error.Error,
2433 "test_POST_upload_named_badfilename",
2435 "name= may not contain a slash",
2437 # make sure that nothing was added
2438 d.addCallback(lambda res:
2439 self.failUnlessNodeKeysAre(self._foo_node,
2440 [u"bar.txt", u"baz.txt", u"blockingfile",
2441 u"empty", u"n\u00fc.txt", u"quux.txt",
2445 def test_POST_FILEURL_check(self):
2446 bar_url = self.public_url + "/foo/bar.txt"
2447 d = self.POST(bar_url, t="check")
2449 self.failUnlessIn("Healthy :", res)
2450 d.addCallback(_check)
2451 redir_url = "http://allmydata.org/TARGET"
2452 def _check2(statuscode, target):
2453 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2454 self.failUnlessReallyEqual(target, redir_url)
2455 d.addCallback(lambda res:
2456 self.shouldRedirect2("test_POST_FILEURL_check",
2460 when_done=redir_url))
2461 d.addCallback(lambda res:
2462 self.POST(bar_url, t="check", return_to=redir_url))
2464 self.failUnlessIn("Healthy :", res)
2465 self.failUnlessIn("Return to file", res)
2466 self.failUnlessIn(redir_url, res)
2467 d.addCallback(_check3)
2469 d.addCallback(lambda res:
2470 self.POST(bar_url, t="check", output="JSON"))
2471 def _check_json(res):
2472 data = simplejson.loads(res)
2473 self.failUnlessIn("storage-index", data)
2474 self.failUnless(data["results"]["healthy"])
2475 d.addCallback(_check_json)
2479 def test_POST_FILEURL_check_and_repair(self):
2480 bar_url = self.public_url + "/foo/bar.txt"
2481 d = self.POST(bar_url, t="check", repair="true")
2483 self.failUnlessIn("Healthy :", res)
2484 d.addCallback(_check)
2485 redir_url = "http://allmydata.org/TARGET"
2486 def _check2(statuscode, target):
2487 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2488 self.failUnlessReallyEqual(target, redir_url)
2489 d.addCallback(lambda res:
2490 self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2493 t="check", repair="true",
2494 when_done=redir_url))
2495 d.addCallback(lambda res:
2496 self.POST(bar_url, t="check", return_to=redir_url))
2498 self.failUnlessIn("Healthy :", res)
2499 self.failUnlessIn("Return to file", res)
2500 self.failUnlessIn(redir_url, res)
2501 d.addCallback(_check3)
2504 def test_POST_DIRURL_check(self):
2505 foo_url = self.public_url + "/foo/"
2506 d = self.POST(foo_url, t="check")
2508 self.failUnlessIn("Healthy :", res)
2509 d.addCallback(_check)
2510 redir_url = "http://allmydata.org/TARGET"
2511 def _check2(statuscode, target):
2512 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2513 self.failUnlessReallyEqual(target, redir_url)
2514 d.addCallback(lambda res:
2515 self.shouldRedirect2("test_POST_DIRURL_check",
2519 when_done=redir_url))
2520 d.addCallback(lambda res:
2521 self.POST(foo_url, t="check", return_to=redir_url))
2523 self.failUnlessIn("Healthy :", res)
2524 self.failUnlessIn("Return to file/directory", res)
2525 self.failUnlessIn(redir_url, res)
2526 d.addCallback(_check3)
2528 d.addCallback(lambda res:
2529 self.POST(foo_url, t="check", output="JSON"))
2530 def _check_json(res):
2531 data = simplejson.loads(res)
2532 self.failUnlessIn("storage-index", data)
2533 self.failUnless(data["results"]["healthy"])
2534 d.addCallback(_check_json)
2538 def test_POST_DIRURL_check_and_repair(self):
2539 foo_url = self.public_url + "/foo/"
2540 d = self.POST(foo_url, t="check", repair="true")
2542 self.failUnlessIn("Healthy :", res)
2543 d.addCallback(_check)
2544 redir_url = "http://allmydata.org/TARGET"
2545 def _check2(statuscode, target):
2546 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2547 self.failUnlessReallyEqual(target, redir_url)
2548 d.addCallback(lambda res:
2549 self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2552 t="check", repair="true",
2553 when_done=redir_url))
2554 d.addCallback(lambda res:
2555 self.POST(foo_url, t="check", return_to=redir_url))
2557 self.failUnlessIn("Healthy :", res)
2558 self.failUnlessIn("Return to file/directory", res)
2559 self.failUnlessIn(redir_url, res)
2560 d.addCallback(_check3)
2563 def test_POST_FILEURL_mdmf_check(self):
2564 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2565 d = self.POST(quux_url, t="check")
2567 self.failUnlessIn("Healthy", res)
2568 d.addCallback(_check)
2569 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2570 d.addCallback(lambda ignored:
2571 self.POST(quux_extension_url, t="check"))
2572 d.addCallback(_check)
2575 def test_POST_FILEURL_mdmf_check_and_repair(self):
2576 quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2577 d = self.POST(quux_url, t="check", repair="true")
2579 self.failUnlessIn("Healthy", res)
2580 d.addCallback(_check)
2581 quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2582 d.addCallback(lambda ignored:
2583 self.POST(quux_extension_url, t="check", repair="true"))
2584 d.addCallback(_check)
2587 def wait_for_operation(self, ignored, ophandle):
2588 url = "/operations/" + ophandle
2589 url += "?t=status&output=JSON"
2592 data = simplejson.loads(res)
2593 if not data["finished"]:
2594 d = self.stall(delay=1.0)
2595 d.addCallback(self.wait_for_operation, ophandle)
2601 def get_operation_results(self, ignored, ophandle, output=None):
2602 url = "/operations/" + ophandle
2605 url += "&output=" + output
2608 if output and output.lower() == "json":
2609 return simplejson.loads(res)
2614 def test_POST_DIRURL_deepcheck_no_ophandle(self):
2615 d = self.shouldFail2(error.Error,
2616 "test_POST_DIRURL_deepcheck_no_ophandle",
2618 "slow operation requires ophandle=",
2619 self.POST, self.public_url, t="start-deep-check")
2622 def test_POST_DIRURL_deepcheck(self):
2623 def _check_redirect(statuscode, target):
2624 self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2625 self.failUnless(target.endswith("/operations/123"))
2626 d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2627 self.POST, self.public_url,
2628 t="start-deep-check", ophandle="123")
2629 d.addCallback(self.wait_for_operation, "123")
2630 def _check_json(data):
2631 self.failUnlessReallyEqual(data["finished"], True)
2632 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2633 self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2634 d.addCallback(_check_json)
2635 d.addCallback(self.get_operation_results, "123", "html")
2636 def _check_html(res):
2637 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2638 self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2639 self.failUnlessIn(FAVICON_MARKUP, res)
2640 d.addCallback(_check_html)
2642 d.addCallback(lambda res:
2643 self.GET("/operations/123/"))
2644 d.addCallback(_check_html) # should be the same as without the slash
2646 d.addCallback(lambda res:
2647 self.shouldFail2(error.Error, "one", "404 Not Found",
2648 "No detailed results for SI bogus",
2649 self.GET, "/operations/123/bogus"))
2651 foo_si = self._foo_node.get_storage_index()
2652 foo_si_s = base32.b2a(foo_si)
2653 d.addCallback(lambda res:
2654 self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2655 def _check_foo_json(res):
2656 data = simplejson.loads(res)
2657 self.failUnlessEqual(data["storage-index"], foo_si_s)
2658 self.failUnless(data["results"]["healthy"])
2659 d.addCallback(_check_foo_json)
2662 def test_POST_DIRURL_deepcheck_and_repair(self):
2663 d = self.POST(self.public_url, t="start-deep-check", repair="true",
2664 ophandle="124", output="json", followRedirect=True)
2665 d.addCallback(self.wait_for_operation, "124")
2666 def _check_json(data):
2667 self.failUnlessReallyEqual(data["finished"], True)
2668 self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2669 self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2670 self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2671 self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2672 self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2673 self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2674 self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2675 self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2676 self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2677 self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2678 d.addCallback(_check_json)
2679 d.addCallback(self.get_operation_results, "124", "html")
2680 def _check_html(res):
2681 self.failUnlessIn("Objects Checked: <span>10</span>", res)
2683 self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2684 self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2685 self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2687 self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2688 self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2689 self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2691 self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2692 self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2693 self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2695 self.failUnlessIn(FAVICON_MARKUP, res)
2696 d.addCallback(_check_html)
2699 def test_POST_FILEURL_bad_t(self):
2700 d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2701 "POST to file: bad t=bogus",
2702 self.POST, self.public_url + "/foo/bar.txt",
2706 def test_POST_mkdir(self): # return value?
2707 d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2708 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2709 d.addCallback(self.failUnlessNodeKeysAre, [])
2712 def test_POST_mkdir_mdmf(self):
2713 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2714 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715 d.addCallback(lambda node:
2716 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2719 def test_POST_mkdir_sdmf(self):
2720 d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2721 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2722 d.addCallback(lambda node:
2723 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2726 def test_POST_mkdir_bad_format(self):
2727 return self.shouldHTTPError("POST_mkdir_bad_format",
2728 400, "Bad Request", "Unknown format: foo",
2729 self.POST, self.public_url +
2730 "/foo?t=mkdir&name=newdir&format=foo")
2732 def test_POST_mkdir_initial_children(self):
2733 (newkids, caps) = self._create_initial_children()
2734 d = self.POST2(self.public_url +
2735 "/foo?t=mkdir-with-children&name=newdir",
2736 simplejson.dumps(newkids))
2737 d.addCallback(lambda res:
2738 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2739 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2741 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2742 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2745 def test_POST_mkdir_initial_children_mdmf(self):
2746 (newkids, caps) = self._create_initial_children()
2747 d = self.POST2(self.public_url +
2748 "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2749 simplejson.dumps(newkids))
2750 d.addCallback(lambda res:
2751 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753 d.addCallback(lambda node:
2754 self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2755 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2761 def test_POST_mkdir_initial_children_sdmf(self):
2762 (newkids, caps) = self._create_initial_children()
2763 d = self.POST2(self.public_url +
2764 "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2765 simplejson.dumps(newkids))
2766 d.addCallback(lambda res:
2767 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2768 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2769 d.addCallback(lambda node:
2770 self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2771 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2772 d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2776 def test_POST_mkdir_initial_children_bad_format(self):
2777 (newkids, caps) = self._create_initial_children()
2778 return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2779 400, "Bad Request", "Unknown format: foo",
2780 self.POST, self.public_url + \
2781 "/foo?t=mkdir-with-children&name=newdir&format=foo",
2782 simplejson.dumps(newkids))
2784 def test_POST_mkdir_immutable(self):
2785 (newkids, caps) = self._create_immutable_children()
2786 d = self.POST2(self.public_url +
2787 "/foo?t=mkdir-immutable&name=newdir",
2788 simplejson.dumps(newkids))
2789 d.addCallback(lambda res:
2790 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2791 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2792 d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2793 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2794 d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2795 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2796 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2797 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2798 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2799 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2800 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2801 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2802 d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2805 def test_POST_mkdir_immutable_bad(self):
2806 (newkids, caps) = self._create_initial_children()
2807 d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2809 "needed to be immutable but was not",
2812 "/foo?t=mkdir-immutable&name=newdir",
2813 simplejson.dumps(newkids))
2816 def test_POST_mkdir_2(self):
2817 d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2818 d.addCallback(lambda res:
2819 self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2820 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2821 d.addCallback(self.failUnlessNodeKeysAre, [])
2824 def test_POST_mkdirs_2(self):
2825 d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2826 d.addCallback(lambda res:
2827 self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2828 d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2829 d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2830 d.addCallback(self.failUnlessNodeKeysAre, [])
2833 def test_POST_mkdir_no_parentdir_noredirect(self):
2834 d = self.POST("/uri?t=mkdir")
2835 def _after_mkdir(res):
2836 uri.DirectoryURI.init_from_string(res)
2837 d.addCallback(_after_mkdir)
2840 def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2841 d = self.POST("/uri?t=mkdir&format=mdmf")
2842 def _after_mkdir(res):
2843 u = uri.from_string(res)
2844 # Check that this is an MDMF writecap
2845 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2846 d.addCallback(_after_mkdir)
2849 def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2850 d = self.POST("/uri?t=mkdir&format=sdmf")
2851 def _after_mkdir(res):
2852 u = uri.from_string(res)
2853 self.failUnlessIsInstance(u, uri.DirectoryURI)
2854 d.addCallback(_after_mkdir)
2857 def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2858 return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2859 400, "Bad Request", "Unknown format: foo",
2860 self.POST, self.public_url +
2861 "/uri?t=mkdir&format=foo")
2863 def test_POST_mkdir_no_parentdir_noredirect2(self):
2864 # make sure form-based arguments (as on the welcome page) still work
2865 d = self.POST("/uri", t="mkdir")
2866 def _after_mkdir(res):
2867 uri.DirectoryURI.init_from_string(res)
2868 d.addCallback(_after_mkdir)
2869 d.addErrback(self.explain_web_error)
2872 def test_POST_mkdir_no_parentdir_redirect(self):
2873 d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2874 d.addBoth(self.shouldRedirect, None, statuscode='303')
2875 def _check_target(target):
2876 target = urllib.unquote(target)
2877 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2878 d.addCallback(_check_target)
2881 def test_POST_mkdir_no_parentdir_redirect2(self):
2882 d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2883 d.addBoth(self.shouldRedirect, None, statuscode='303')
2884 def _check_target(target):
2885 target = urllib.unquote(target)
2886 self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2887 d.addCallback(_check_target)
2888 d.addErrback(self.explain_web_error)
2891 def _make_readonly(self, u):
2892 ro_uri = uri.from_string(u).get_readonly()
2895 return ro_uri.to_string()
2897 def _create_initial_children(self):
2898 contents, n, filecap1 = self.makefile(12)
2899 md1 = {"metakey1": "metavalue1"}
2900 filecap2 = make_mutable_file_uri()
2901 node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2902 filecap3 = node3.get_readonly_uri()
2903 node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2904 dircap = DirectoryNode(node4, None, None).get_uri()
2905 mdmfcap = make_mutable_file_uri(mdmf=True)
2906 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2907 emptydircap = "URI:DIR2-LIT:"
2908 newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1,
2909 "ro_uri": self._make_readonly(filecap1),
2910 "metadata": md1, }],
2911 u"child-mutable": ["filenode", {"rw_uri": filecap2,
2912 "ro_uri": self._make_readonly(filecap2)}],
2913 u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2914 u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap,
2915 "ro_uri": unknown_rocap}],
2916 u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}],
2917 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2918 u"dirchild": ["dirnode", {"rw_uri": dircap,
2919 "ro_uri": self._make_readonly(dircap)}],
2920 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2921 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2922 u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2923 "ro_uri": self._make_readonly(mdmfcap)}],
2925 return newkids, {'filecap1': filecap1,
2926 'filecap2': filecap2,
2927 'filecap3': filecap3,
2928 'unknown_rwcap': unknown_rwcap,
2929 'unknown_rocap': unknown_rocap,
2930 'unknown_immcap': unknown_immcap,
2932 'litdircap': litdircap,
2933 'emptydircap': emptydircap,
2936 def _create_immutable_children(self):
2937 contents, n, filecap1 = self.makefile(12)
2938 md1 = {"metakey1": "metavalue1"}
2939 tnode = create_chk_filenode("immutable directory contents\n"*10,
2940 self.get_all_contents())
2941 dnode = DirectoryNode(tnode, None, None)
2942 assert not dnode.is_mutable()
2943 immdircap = dnode.get_uri()
2944 litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2945 emptydircap = "URI:DIR2-LIT:"
2946 newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2947 "metadata": md1, }],
2948 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}],
2949 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2950 u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}],
2951 u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}],
2953 return newkids, {'filecap1': filecap1,
2954 'unknown_immcap': unknown_immcap,
2955 'immdircap': immdircap,
2956 'litdircap': litdircap,
2957 'emptydircap': emptydircap}
2959 def test_POST_mkdir_no_parentdir_initial_children(self):
2960 (newkids, caps) = self._create_initial_children()
2961 d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2962 def _after_mkdir(res):
2963 self.failUnless(res.startswith("URI:DIR"), res)
2964 n = self.s.create_node_from_uri(res)
2965 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2966 d2.addCallback(lambda ign:
2967 self.failUnlessROChildURIIs(n, u"child-imm",
2969 d2.addCallback(lambda ign:
2970 self.failUnlessRWChildURIIs(n, u"child-mutable",
2972 d2.addCallback(lambda ign:
2973 self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2975 d2.addCallback(lambda ign:
2976 self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2977 caps['unknown_rwcap']))
2978 d2.addCallback(lambda ign:
2979 self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2980 caps['unknown_rocap']))
2981 d2.addCallback(lambda ign:
2982 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2983 caps['unknown_immcap']))
2984 d2.addCallback(lambda ign:
2985 self.failUnlessRWChildURIIs(n, u"dirchild",
2988 d.addCallback(_after_mkdir)
2991 def test_POST_mkdir_no_parentdir_unexpected_children(self):
2992 # the regular /uri?t=mkdir operation is specified to ignore its body.
2993 # Only t=mkdir-with-children pays attention to it.
2994 (newkids, caps) = self._create_initial_children()
2995 d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2997 "t=mkdir does not accept children=, "
2998 "try t=mkdir-with-children instead",
2999 self.POST2, "/uri?t=mkdir", # without children
3000 simplejson.dumps(newkids))
3003 def test_POST_noparent_bad(self):
3004 d = self.shouldHTTPError("POST_noparent_bad",
3006 "/uri accepts only PUT, PUT?t=mkdir, "
3007 "POST?t=upload, and POST?t=mkdir",
3008 self.POST, "/uri?t=bogus")
3011 def test_POST_mkdir_no_parentdir_immutable(self):
3012 (newkids, caps) = self._create_immutable_children()
3013 d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3014 def _after_mkdir(res):
3015 self.failUnless(res.startswith("URI:DIR"), res)
3016 n = self.s.create_node_from_uri(res)
3017 d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3018 d2.addCallback(lambda ign:
3019 self.failUnlessROChildURIIs(n, u"child-imm",
3021 d2.addCallback(lambda ign:
3022 self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3023 caps['unknown_immcap']))
3024 d2.addCallback(lambda ign:
3025 self.failUnlessROChildURIIs(n, u"dirchild-imm",
3027 d2.addCallback(lambda ign:
3028 self.failUnlessROChildURIIs(n, u"dirchild-lit",
3030 d2.addCallback(lambda ign:
3031 self.failUnlessROChildURIIs(n, u"dirchild-empty",
3032 caps['emptydircap']))
3034 d.addCallback(_after_mkdir)
3037 def test_POST_mkdir_no_parentdir_immutable_bad(self):
3038 (newkids, caps) = self._create_initial_children()
3039 d = self.shouldFail2(error.Error,
3040 "test_POST_mkdir_no_parentdir_immutable_bad",
3042 "needed to be immutable but was not",
3044 "/uri?t=mkdir-immutable",
3045 simplejson.dumps(newkids))
3048 def test_welcome_page_mkdir_button(self):
3049 # Fetch the welcome page.
3051 def _after_get_welcome_page(res):
3052 MKDIR_BUTTON_RE = re.compile(
3053 '<form action="([^"]*)" method="post".*?'
3054 '<input type="hidden" name="t" value="([^"]*)" />'
3055 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3056 '<input type="submit" value="Create a directory" />',
3058 mo = MKDIR_BUTTON_RE.search(res)
3059 formaction = mo.group(1)
3061 formaname = mo.group(3)
3062 formavalue = mo.group(4)
3063 return (formaction, formt, formaname, formavalue)
3064 d.addCallback(_after_get_welcome_page)
3065 def _after_parse_form(res):
3066 (formaction, formt, formaname, formavalue) = res
3067 return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3068 d.addCallback(_after_parse_form)
3069 d.addBoth(self.shouldRedirect, None, statuscode='303')
3072 def test_POST_mkdir_replace(self): # return value?
3073 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3074 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3075 d.addCallback(self.failUnlessNodeKeysAre, [])
3078 def test_POST_mkdir_no_replace_queryarg(self): # return value?
3079 d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3080 d.addBoth(self.shouldFail, error.Error,
3081 "POST_mkdir_no_replace_queryarg",
3083 "There was already a child by that name, and you asked me "
3084 "to not replace it")
3085 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3086 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3089 def test_POST_mkdir_no_replace_field(self): # return value?
3090 d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3092 d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3094 "There was already a child by that name, and you asked me "
3095 "to not replace it")
3096 d.addCallback(lambda res: self._foo_node.get(u"sub"))
3097 d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3100 def test_POST_mkdir_whendone_field(self):
3101 d = self.POST(self.public_url + "/foo",
3102 t="mkdir", name="newdir", when_done="/THERE")
3103 d.addBoth(self.shouldRedirect, "/THERE")
3104 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3105 d.addCallback(self.failUnlessNodeKeysAre, [])
3108 def test_POST_mkdir_whendone_queryarg(self):
3109 d = self.POST(self.public_url + "/foo?when_done=/THERE",
3110 t="mkdir", name="newdir")
3111 d.addBoth(self.shouldRedirect, "/THERE")
3112 d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3113 d.addCallback(self.failUnlessNodeKeysAre, [])
3116 def test_POST_bad_t(self):
3117 d = self.shouldFail2(error.Error, "POST_bad_t",
3119 "POST to a directory with bad t=BOGUS",
3120 self.POST, self.public_url + "/foo", t="BOGUS")
3123 def test_POST_set_children(self, command_name="set_children"):
3124 contents9, n9, newuri9 = self.makefile(9)
3125 contents10, n10, newuri10 = self.makefile(10)
3126 contents11, n11, newuri11 = self.makefile(11)
3129 "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3132 "ctime": 1002777696.7564139,
3133 "mtime": 1002777696.7564139
3136 "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3139 "ctime": 1002777696.7564139,
3140 "mtime": 1002777696.7564139
3143 "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3146 "ctime": 1002777696.7564139,
3147 "mtime": 1002777696.7564139
3150 }""" % (newuri9, newuri10, newuri11)
3152 url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3154 d = client.getPage(url, method="POST", postdata=reqbody)
3156 self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3157 self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3158 self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3160 d.addCallback(_then)
3161 d.addErrback(self.dump_error)
3164 def test_POST_set_children_with_hyphen(self):
3165 return self.test_POST_set_children(command_name="set-children")
3167 def test_POST_link_uri(self):
3168 contents, n, newuri = self.makefile(8)
3169 d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3170 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3171 d.addCallback(lambda res:
3172 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3176 def test_POST_link_uri_replace(self):
3177 contents, n, newuri = self.makefile(8)
3178 d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3179 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3180 d.addCallback(lambda res:
3181 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3185 def test_POST_link_uri_unknown_bad(self):
3186 d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3187 d.addBoth(self.shouldFail, error.Error,
3188 "POST_link_uri_unknown_bad",
3190 "unknown cap in a write slot")
3193 def test_POST_link_uri_unknown_ro_good(self):
3194 d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3195 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3198 def test_POST_link_uri_unknown_imm_good(self):
3199 d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3200 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3203 def test_POST_link_uri_no_replace_queryarg(self):
3204 contents, n, newuri = self.makefile(8)
3205 d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3206 name="bar.txt", uri=newuri)
3207 d.addBoth(self.shouldFail, error.Error,
3208 "POST_link_uri_no_replace_queryarg",
3210 "There was already a child by that name, and you asked me "
3211 "to not replace it")
3212 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3213 d.addCallback(self.failUnlessIsBarDotTxt)
3216 def test_POST_link_uri_no_replace_field(self):
3217 contents, n, newuri = self.makefile(8)
3218 d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3219 name="bar.txt", uri=newuri)
3220 d.addBoth(self.shouldFail, error.Error,
3221 "POST_link_uri_no_replace_field",
3223 "There was already a child by that name, and you asked me "
3224 "to not replace it")
3225 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3226 d.addCallback(self.failUnlessIsBarDotTxt)
3229 def test_POST_delete(self, command_name='delete'):
3230 d = self._foo_node.list()
3231 def _check_before(children):
3232 self.failUnlessIn(u"bar.txt", children)
3233 d.addCallback(_check_before)
3234 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3235 d.addCallback(lambda res: self._foo_node.list())
3236 def _check_after(children):
3237 self.failIfIn(u"bar.txt", children)
3238 d.addCallback(_check_after)
3241 def test_POST_unlink(self):
3242 return self.test_POST_delete(command_name='unlink')
3244 def test_POST_rename_file(self):
3245 d = self.POST(self.public_url + "/foo", t="rename",
3246 from_name="bar.txt", to_name='wibble.txt')
3247 d.addCallback(lambda res:
3248 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3249 d.addCallback(lambda res:
3250 self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3251 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3252 d.addCallback(self.failUnlessIsBarDotTxt)
3253 d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3254 d.addCallback(self.failUnlessIsBarJSON)
3257 def test_POST_rename_file_redundant(self):
3258 d = self.POST(self.public_url + "/foo", t="rename",
3259 from_name="bar.txt", to_name='bar.txt')
3260 d.addCallback(lambda res:
3261 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3262 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3263 d.addCallback(self.failUnlessIsBarDotTxt)
3264 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3265 d.addCallback(self.failUnlessIsBarJSON)
3268 def test_POST_rename_file_replace(self):
3269 # rename a file and replace a directory with it
3270 d = self.POST(self.public_url + "/foo", t="rename",
3271 from_name="bar.txt", to_name='empty')
3272 d.addCallback(lambda res:
3273 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3274 d.addCallback(lambda res:
3275 self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3276 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3277 d.addCallback(self.failUnlessIsBarDotTxt)
3278 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3279 d.addCallback(self.failUnlessIsBarJSON)
3282 def test_POST_rename_file_no_replace_queryarg(self):
3283 # rename a file and replace a directory with it
3284 d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3285 from_name="bar.txt", to_name='empty')
3286 d.addBoth(self.shouldFail, error.Error,
3287 "POST_rename_file_no_replace_queryarg",
3289 "There was already a child by that name, and you asked me "
3290 "to not replace it")
3291 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3292 d.addCallback(self.failUnlessIsEmptyJSON)
3295 def test_POST_rename_file_no_replace_field(self):
3296 # rename a file and replace a directory with it
3297 d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3298 from_name="bar.txt", to_name='empty')
3299 d.addBoth(self.shouldFail, error.Error,
3300 "POST_rename_file_no_replace_field",
3302 "There was already a child by that name, and you asked me "
3303 "to not replace it")
3304 d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3305 d.addCallback(self.failUnlessIsEmptyJSON)
3308 def failUnlessIsEmptyJSON(self, res):
3309 data = simplejson.loads(res)
3310 self.failUnlessEqual(data[0], "dirnode", data)
3311 self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3313 def test_POST_rename_file_slash_fail(self):
3314 d = self.POST(self.public_url + "/foo", t="rename",
3315 from_name="bar.txt", to_name='kirk/spock.txt')
3316 d.addBoth(self.shouldFail, error.Error,
3317 "test_POST_rename_file_slash_fail",
3319 "to_name= may not contain a slash",
3321 d.addCallback(lambda res:
3322 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3325 def test_POST_rename_dir(self):
3326 d = self.POST(self.public_url, t="rename",
3327 from_name="foo", to_name='plunk')
3328 d.addCallback(lambda res:
3329 self.failIfNodeHasChild(self.public_root, u"foo"))
3330 d.addCallback(lambda res:
3331 self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3332 d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3333 d.addCallback(self.failUnlessIsFooJSON)
3336 def test_POST_move_file(self):
3337 d = self.POST(self.public_url + "/foo", t="move",
3338 from_name="bar.txt", to_dir="sub")
3339 d.addCallback(lambda res:
3340 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3341 d.addCallback(lambda res:
3342 self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3343 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3344 d.addCallback(self.failUnlessIsBarDotTxt)
3345 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3346 d.addCallback(self.failUnlessIsBarJSON)
3349 def test_POST_move_file_new_name(self):
3350 d = self.POST(self.public_url + "/foo", t="move",
3351 from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3352 d.addCallback(lambda res:
3353 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3354 d.addCallback(lambda res:
3355 self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3356 d.addCallback(lambda res:
3357 self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3358 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3359 d.addCallback(self.failUnlessIsBarDotTxt)
3360 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3361 d.addCallback(self.failUnlessIsBarJSON)
3364 def test_POST_move_file_replace(self):
3365 d = self.POST(self.public_url + "/foo", t="move",
3366 from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3367 d.addCallback(lambda res:
3368 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3369 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3370 d.addCallback(self.failUnlessIsBarDotTxt)
3371 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3372 d.addCallback(self.failUnlessIsBarJSON)
3375 def test_POST_move_file_no_replace(self):
3376 d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3378 "There was already a child by that name, and you asked me to not replace it",
3379 self.POST, self.public_url + "/foo", t="move",
3380 replace="false", from_name="bar.txt",
3381 to_name="baz.txt", to_dir="sub")
3382 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3383 d.addCallback(self.failUnlessIsBarDotTxt)
3384 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3385 d.addCallback(self.failUnlessIsBarJSON)
3386 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3387 d.addCallback(self.failUnlessIsSubBazDotTxt)
3390 def test_POST_move_file_slash_fail(self):
3391 d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3393 "to_name= may not contain a slash",
3394 self.POST, self.public_url + "/foo", t="move",
3395 from_name="bar.txt",
3396 to_name="slash/fail.txt", to_dir="sub")
3397 d.addCallback(lambda res:
3398 self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3399 d.addCallback(lambda res:
3400 self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3401 d.addCallback(lambda ign:
3402 self.shouldFail2(error.Error,
3403 "test_POST_rename_file_slash_fail2",
3405 "from_name= may not contain a slash",
3406 self.POST, self.public_url + "/foo",
3408 from_name="nope/bar.txt",
3409 to_name="fail.txt", to_dir="sub"))
3412 def test_POST_move_file_no_target(self):
3413 d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3415 "move requires from_name and to_dir",
3416 self.POST, self.public_url + "/foo", t="move",
3417 from_name="bar.txt", to_name="baz.txt")
3418 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3419 d.addCallback(self.failUnlessIsBarDotTxt)
3420 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3421 d.addCallback(self.failUnlessIsBarJSON)
3422 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3423 d.addCallback(self.failUnlessIsBazDotTxt)
3426 def test_POST_move_file_bad_target_type(self):
3427 d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3428 "400 Bad Request", "invalid target_type parameter",
3430 self.public_url + "/foo", t="move",
3431 target_type="*D", from_name="bar.txt",
3435 def test_POST_move_file_multi_level(self):
3436 d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3437 d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3438 from_name="bar.txt", to_dir="sub/level2"))
3439 d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3440 d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3441 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3442 d.addCallback(self.failUnlessIsBarDotTxt)
3443 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3444 d.addCallback(self.failUnlessIsBarJSON)
3447 def test_POST_move_file_to_uri(self):
3448 d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3449 from_name="bar.txt", to_dir=self._sub_uri)
3450 d.addCallback(lambda res:
3451 self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3452 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3453 d.addCallback(self.failUnlessIsBarDotTxt)
3454 d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3455 d.addCallback(self.failUnlessIsBarJSON)
3458 def test_POST_move_file_to_nonexist_dir(self):
3459 d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3460 "404 Not Found", "No such child: nopechucktesta",
3461 self.POST, self.public_url + "/foo", t="move",
3462 from_name="bar.txt", to_dir="nopechucktesta")
3465 def test_POST_move_file_into_file(self):
3466 d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3467 "400 Bad Request", "to_dir is not a directory",
3468 self.POST, self.public_url + "/foo", t="move",
3469 from_name="bar.txt", to_dir="baz.txt")
3470 d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3471 d.addCallback(self.failUnlessIsBazDotTxt)
3472 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3473 d.addCallback(self.failUnlessIsBarDotTxt)
3474 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3475 d.addCallback(self.failUnlessIsBarJSON)
3478 def test_POST_move_file_to_bad_uri(self):
3479 d = self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3480 "400 Bad Request", "to_dir is not a directory",
3481 self.POST, self.public_url + "/foo", t="move",
3482 from_name="bar.txt", target_type="uri",
3483 to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3484 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3485 d.addCallback(self.failUnlessIsBarDotTxt)
3486 d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3487 d.addCallback(self.failUnlessIsBarJSON)
3490 def test_POST_move_dir(self):
3491 d = self.POST(self.public_url + "/foo", t="move",
3492 from_name="bar.txt", to_dir="empty")
3493 d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3494 t="move", from_name="empty", to_dir="sub"))
3495 d.addCallback(lambda res:
3496 self.failIfNodeHasChild(self._foo_node, u"empty"))
3497 d.addCallback(lambda res:
3498 self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3499 d.addCallback(lambda res:
3500 self._sub_node.get_child_at_path(u"empty"))
3501 d.addCallback(lambda node:
3502 self.failUnlessNodeHasChild(node, u"bar.txt"))
3503 d.addCallback(lambda res:
3504 self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3505 d.addCallback(self.failUnlessIsBarDotTxt)
3508 def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3509 """ If target is not None then the redirection has to go to target. If
3510 statuscode is not None then the redirection has to be accomplished with
3511 that HTTP status code."""
3512 if not isinstance(res, failure.Failure):
3513 to_where = (target is None) and "somewhere" or ("to " + target)
3514 self.fail("%s: we were expecting to get redirected %s, not get an"
3515 " actual page: %s" % (which, to_where, res))
3516 res.trap(error.PageRedirect)
3517 if statuscode is not None:
3518 self.failUnlessReallyEqual(res.value.status, statuscode,
3519 "%s: not a redirect" % which)
3520 if target is not None:
3521 # the PageRedirect does not seem to capture the uri= query arg
3522 # properly, so we can't check for it.
3523 realtarget = self.webish_url + target
3524 self.failUnlessReallyEqual(res.value.location, realtarget,
3525 "%s: wrong target" % which)
3526 return res.value.location
3528 def test_GET_URI_form(self):
3529 base = "/uri?uri=%s" % self._bar_txt_uri
3530 # this is supposed to give us a redirect to /uri/$URI, plus arguments
3531 targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3533 d.addBoth(self.shouldRedirect, targetbase)
3534 d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3535 d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3536 d.addCallback(lambda res: self.GET(base+"&t=json"))
3537 d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3538 d.addCallback(self.log, "about to get file by uri")
3539 d.addCallback(lambda res: self.GET(base, followRedirect=True))
3540 d.addCallback(self.failUnlessIsBarDotTxt)
3541 d.addCallback(self.log, "got file by uri, about to get dir by uri")
3542 d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3543 followRedirect=True))
3544 d.addCallback(self.failUnlessIsFooJSON)
3545 d.addCallback(self.log, "got dir by uri")
3549 def test_GET_URI_form_bad(self):
3550 d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3551 "400 Bad Request", "GET /uri requires uri=",
3555 def test_GET_rename_form(self):
3556 d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3557 followRedirect=True)
3559 self.failUnlessIn('name="when_done" value="."', res)
3560 self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3561 self.failUnlessIn(FAVICON_MARKUP, res)
3562 d.addCallback(_check)
3565 def log(self, res, msg):
3566 #print "MSG: %s RES: %s" % (msg, res)
3570 def test_GET_URI_URL(self):
3571 base = "/uri/%s" % self._bar_txt_uri
3573 d.addCallback(self.failUnlessIsBarDotTxt)
3574 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3575 d.addCallback(self.failUnlessIsBarDotTxt)
3576 d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3577 d.addCallback(self.failUnlessIsBarDotTxt)
3580 def test_GET_URI_URL_dir(self):
3581 base = "/uri/%s?t=json" % self._foo_uri
3583 d.addCallback(self.failUnlessIsFooJSON)
3586 def test_GET_URI_URL_missing(self):
3587 base = "/uri/%s" % self._bad_file_uri
3588 d = self.shouldHTTPError("test_GET_URI_URL_missing",
3589 http.GONE, None, "NotEnoughSharesError",
3591 # TODO: how can we exercise both sides of WebDownloadTarget.fail
3592 # here? we must arrange for a download to fail after target.open()
3593 # has been called, and then inspect the response to see that it is
3594 # shorter than we expected.
3597 def test_PUT_DIRURL_uri(self):
3598 d = self.s.create_dirnode()
3600 new_uri = dn.get_uri()
3601 # replace /foo with a new (empty) directory
3602 d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3603 d.addCallback(lambda res:
3604 self.failUnlessReallyEqual(res.strip(), new_uri))
3605 d.addCallback(lambda res:
3606 self.failUnlessRWChildURIIs(self.public_root,
3610 d.addCallback(_made_dir)
3613 def test_PUT_DIRURL_uri_noreplace(self):
3614 d = self.s.create_dirnode()
3616 new_uri = dn.get_uri()
3617 # replace /foo with a new (empty) directory, but ask that
3618 # replace=false, so it should fail
3619 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3620 "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3622 self.public_url + "/foo?t=uri&replace=false",
3624 d.addCallback(lambda res:
3625 self.failUnlessRWChildURIIs(self.public_root,
3629 d.addCallback(_made_dir)
3632 def test_PUT_DIRURL_bad_t(self):
3633 d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3634 "400 Bad Request", "PUT to a directory",
3635 self.PUT, self.public_url + "/foo?t=BOGUS", "")
3636 d.addCallback(lambda res:
3637 self.failUnlessRWChildURIIs(self.public_root,
3642 def test_PUT_NEWFILEURL_uri(self):
3643 contents, n, new_uri = self.makefile(8)
3644 d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3645 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3646 d.addCallback(lambda res:
3647 self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3651 def test_PUT_NEWFILEURL_mdmf(self):
3652 new_contents = self.NEWFILE_CONTENTS * 300000
3653 d = self.PUT(self.public_url + \
3654 "/foo/mdmf.txt?format=mdmf",
3656 d.addCallback(lambda ignored:
3657 self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3658 def _got_json(json):
3659 data = simplejson.loads(json)
3661 self.failUnlessIn("format", data)
3662 self.failUnlessEqual(data["format"], "MDMF")
3663 self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3664 self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3665 d.addCallback(_got_json)
3668 def test_PUT_NEWFILEURL_sdmf(self):
3669 new_contents = self.NEWFILE_CONTENTS * 300000
3670 d = self.PUT(self.public_url + \
3671 "/foo/sdmf.txt?format=sdmf",
3673 d.addCallback(lambda ignored:
3674 self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3675 def _got_json(json):
3676 data = simplejson.loads(json)
3678 self.failUnlessIn("format", data)
3679 self.failUnlessEqual(data["format"], "SDMF")
3680 d.addCallback(_got_json)
3683 def test_PUT_NEWFILEURL_bad_format(self):
3684 new_contents = self.NEWFILE_CONTENTS * 300000
3685 return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3686 400, "Bad Request", "Unknown format: foo",
3687 self.PUT, self.public_url + \
3688 "/foo/foo.txt?format=foo",
3691 def test_PUT_NEWFILEURL_uri_replace(self):
3692 contents, n, new_uri = self.makefile(8)
3693 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3694 d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3695 d.addCallback(lambda res:
3696 self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3700 def test_PUT_NEWFILEURL_uri_no_replace(self):
3701 contents, n, new_uri = self.makefile(8)
3702 d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3703 d.addBoth(self.shouldFail, error.Error,
3704 "PUT_NEWFILEURL_uri_no_replace",
3706 "There was already a child by that name, and you asked me "
3707 "to not replace it")
3710 def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3711 d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3712 d.addBoth(self.shouldFail, error.Error,
3713 "POST_put_uri_unknown_bad",
3715 "unknown cap in a write slot")
3718 def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3719 d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3720 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3721 u"put-future-ro.txt")
3724 def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3725 d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3726 d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3727 u"put-future-imm.txt")
3730 def test_PUT_NEWFILE_URI(self):
3731 file_contents = "New file contents here\n"
3732 d = self.PUT("/uri", file_contents)
3734 assert isinstance(uri, str), uri
3735 self.failUnlessIn(uri, self.get_all_contents())
3736 self.failUnlessReallyEqual(self.get_all_contents()[uri],
3738 return self.GET("/uri/%s" % uri)
3739 d.addCallback(_check)
3741 self.failUnlessReallyEqual(res, file_contents)
3742 d.addCallback(_check2)
3745 def test_PUT_NEWFILE_URI_not_mutable(self):
3746 file_contents = "New file contents here\n"
3747 d = self.PUT("/uri?mutable=false", file_contents)
3749 assert isinstance(uri, str), uri
3750 self.failUnlessIn(uri, self.get_all_contents())
3751 self.failUnlessReallyEqual(self.get_all_contents()[uri],
3753 return self.GET("/uri/%s" % uri)
3754 d.addCallback(_check)
3756 self.failUnlessReallyEqual(res, file_contents)
3757 d.addCallback(_check2)
3760 def test_PUT_NEWFILE_URI_only_PUT(self):
3761 d = self.PUT("/uri?t=bogus", "")
3762 d.addBoth(self.shouldFail, error.Error,
3763 "PUT_NEWFILE_URI_only_PUT",
3765 "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3768 def test_PUT_NEWFILE_URI_mutable(self):
3769 file_contents = "New file contents here\n"
3770 d = self.PUT("/uri?mutable=true", file_contents)
3771 def _check1(filecap):
3772 filecap = filecap.strip()
3773 self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3774 self.filecap = filecap
3775 u = uri.WriteableSSKFileURI.init_from_string(filecap)
3776 self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3777 n = self.s.create_node_from_uri(filecap)
3778 return n.download_best_version()
3779 d.addCallback(_check1)
3781 self.failUnlessReallyEqual(data, file_contents)
3782 return self.GET("/uri/%s" % urllib.quote(self.filecap))
3783 d.addCallback(_check2)
3785 self.failUnlessReallyEqual(res, file_contents)
3786 d.addCallback(_check3)
3789 def test_PUT_mkdir(self):
3790 d = self.PUT("/uri?t=mkdir", "")
3792 n = self.s.create_node_from_uri(uri.strip())
3793 d2 = self.failUnlessNodeKeysAre(n, [])
3794 d2.addCallback(lambda res:
3795 self.GET("/uri/%s?t=json" % uri))
3797 d.addCallback(_check)
3798 d.addCallback(self.failUnlessIsEmptyJSON)
3801 def test_PUT_mkdir_mdmf(self):
3802 d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3804 u = uri.from_string(res)
3805 # Check that this is an MDMF writecap
3806 self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3810 def test_PUT_mkdir_sdmf(self):
3811 d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3813 u = uri.from_string(res)
3814 self.failUnlessIsInstance(u, uri.DirectoryURI)
3818 def test_PUT_mkdir_bad_format(self):
3819 return self.shouldHTTPError("PUT_mkdir_bad_format",
3820 400, "Bad Request", "Unknown format: foo",
3821 self.PUT, "/uri?t=mkdir&format=foo",
3824 def test_POST_check(self):
3825 d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3827 # this returns a string form of the results, which are probably
3828 # None since we're using fake filenodes.
3829 # TODO: verify that the check actually happened, by changing
3830 # FakeCHKFileNode to count how many times .check() has been
3833 d.addCallback(_done)
3837 def test_PUT_update_at_offset(self):
3838 file_contents = "test file" * 100000 # about 900 KiB
3839 d = self.PUT("/uri?mutable=true", file_contents)
3841 self.filecap = filecap
3842 new_data = file_contents[:100]
3843 new = "replaced and so on"
3845 new_data += file_contents[len(new_data):]
3846 assert len(new_data) == len(file_contents)
3847 self.new_data = new_data
3848 d.addCallback(_then)
3849 d.addCallback(lambda ignored:
3850 self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3851 "replaced and so on"))
3852 def _get_data(filecap):
3853 n = self.s.create_node_from_uri(filecap)
3854 return n.download_best_version()
3855 d.addCallback(_get_data)
3856 d.addCallback(lambda results:
3857 self.failUnlessEqual(results, self.new_data))
3858 # Now try appending things to the file
3859 d.addCallback(lambda ignored:
3860 self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3862 d.addCallback(_get_data)
3863 d.addCallback(lambda results:
3864 self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3865 # and try replacing the beginning of the file
3866 d.addCallback(lambda ignored:
3867 self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3868 d.addCallback(_get_data)
3869 d.addCallback(lambda results:
3870 self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3873 def test_PUT_update_at_invalid_offset(self):
3874 file_contents = "test file" * 100000 # about 900 KiB
3875 d = self.PUT("/uri?mutable=true", file_contents)
3877 self.filecap = filecap
3878 d.addCallback(_then)
3879 # Negative offsets should cause an error.
3880 d.addCallback(lambda ignored:
3881 self.shouldHTTPError("PUT_update_at_invalid_offset",
3885 "/uri/%s?offset=-1" % self.filecap,
3889 def test_PUT_update_at_offset_immutable(self):
3890 file_contents = "Test file" * 100000
3891 d = self.PUT("/uri", file_contents)
3893 self.filecap = filecap
3894 d.addCallback(_then)
3895 d.addCallback(lambda ignored:
3896 self.shouldHTTPError("PUT_update_at_offset_immutable",
3900 "/uri/%s?offset=50" % self.filecap,
3905 def test_bad_method(self):
3906 url = self.webish_url + self.public_url + "/foo/bar.txt"
3907 d = self.shouldHTTPError("bad_method",
3908 501, "Not Implemented",
3909 "I don't know how to treat a BOGUS request.",
3910 client.getPage, url, method="BOGUS")
3913 def test_short_url(self):
3914 url = self.webish_url + "/uri"
3915 d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3916 "I don't know how to treat a DELETE request.",
3917 client.getPage, url, method="DELETE")
3920 def test_ophandle_bad(self):
3921 url = self.webish_url + "/operations/bogus?t=status"
3922 d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3923 "unknown/expired handle 'bogus'",
3924 client.getPage, url)
3927 def test_ophandle_cancel(self):
3928 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3929 followRedirect=True)
3930 d.addCallback(lambda ignored:
3931 self.GET("/operations/128?t=status&output=JSON"))
3933 data = simplejson.loads(res)
3934 self.failUnless("finished" in data, res)
3935 monitor = self.ws.root.child_operations.handles["128"][0]
3936 d = self.POST("/operations/128?t=cancel&output=JSON")
3938 data = simplejson.loads(res)
3939 self.failUnless("finished" in data, res)
3940 # t=cancel causes the handle to be forgotten
3941 self.failUnless(monitor.is_cancelled())
3942 d.addCallback(_check2)
3944 d.addCallback(_check1)
3945 d.addCallback(lambda ignored:
3946 self.shouldHTTPError("ophandle_cancel",
3947 404, "404 Not Found",
3948 "unknown/expired handle '128'",
3950 "/operations/128?t=status&output=JSON"))
3953 def test_ophandle_retainfor(self):
3954 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3955 followRedirect=True)
3956 d.addCallback(lambda ignored:
3957 self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3959 data = simplejson.loads(res)
3960 self.failUnless("finished" in data, res)
3961 d.addCallback(_check1)
3962 # the retain-for=0 will cause the handle to be expired very soon
3963 d.addCallback(lambda ign:
3964 self.clock.advance(2.0))
3965 d.addCallback(lambda ignored:
3966 self.shouldHTTPError("ophandle_retainfor",
3967 404, "404 Not Found",
3968 "unknown/expired handle '129'",
3970 "/operations/129?t=status&output=JSON"))
3973 def test_ophandle_release_after_complete(self):
3974 d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3975 followRedirect=True)
3976 d.addCallback(self.wait_for_operation, "130")
3977 d.addCallback(lambda ignored:
3978 self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3979 # the release-after-complete=true will cause the handle to be expired
3980 d.addCallback(lambda ignored:
3981 self.shouldHTTPError("ophandle_release_after_complete",
3982 404, "404 Not Found",
3983 "unknown/expired handle '130'",
3985 "/operations/130?t=status&output=JSON"))
3988 def test_uncollected_ophandle_expiration(self):
3989 # uncollected ophandles should expire after 4 days
3990 def _make_uncollected_ophandle(ophandle):
3991 d = self.POST(self.public_url +
3992 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3993 followRedirect=False)
3994 # When we start the operation, the webapi server will want
3995 # to redirect us to the page for the ophandle, so we get
3996 # confirmation that the operation has started. If the
3997 # manifest operation has finished by the time we get there,
3998 # following that redirect (by setting followRedirect=True
3999 # above) has the side effect of collecting the ophandle that
4000 # we've just created, which means that we can't use the
4001 # ophandle to test the uncollected timeout anymore. So,
4002 # instead, catch the 302 here and don't follow it.
4003 d.addBoth(self.should302, "uncollected_ophandle_creation")
4005 # Create an ophandle, don't collect it, then advance the clock by
4006 # 4 days - 1 second and make sure that the ophandle is still there.
4007 d = _make_uncollected_ophandle(131)
4008 d.addCallback(lambda ign:
4009 self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4010 d.addCallback(lambda ign:
4011 self.GET("/operations/131?t=status&output=JSON"))
4013 data = simplejson.loads(res)
4014 self.failUnless("finished" in data, res)
4015 d.addCallback(_check1)
4016 # Create an ophandle, don't collect it, then try to collect it
4017 # after 4 days. It should be gone.
4018 d.addCallback(lambda ign:
4019 _make_uncollected_ophandle(132))
4020 d.addCallback(lambda ign:
4021 self.clock.advance(96*60*60))
4022 d.addCallback(lambda ign:
4023 self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4024 404, "404 Not Found",
4025 "unknown/expired handle '132'",
4027 "/operations/132?t=status&output=JSON"))
4030 def test_collected_ophandle_expiration(self):
4031 # collected ophandles should expire after 1 day
4032 def _make_collected_ophandle(ophandle):
4033 d = self.POST(self.public_url +
4034 "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4035 followRedirect=True)
4036 # By following the initial redirect, we collect the ophandle
4037 # we've just created.
4039 # Create a collected ophandle, then collect it after 23 hours
4040 # and 59 seconds to make sure that it is still there.
4041 d = _make_collected_ophandle(133)
4042 d.addCallback(lambda ign:
4043 self.clock.advance((24*60*60) - 1))
4044 d.addCallback(lambda ign:
4045 self.GET("/operations/133?t=status&output=JSON"))
4047 data = simplejson.loads(res)
4048 self.failUnless("finished" in data, res)
4049 d.addCallback(_check1)
4050 # Create another uncollected ophandle, then try to collect it
4051 # after 24 hours to make sure that it is gone.
4052 d.addCallback(lambda ign:
4053 _make_collected_ophandle(134))
4054 d.addCallback(lambda ign:
4055 self.clock.advance(24*60*60))
4056 d.addCallback(lambda ign:
4057 self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4058 404, "404 Not Found",
4059 "unknown/expired handle '134'",
4061 "/operations/134?t=status&output=JSON"))
4064 def test_incident(self):
4065 d = self.POST("/report_incident", details="eek")
4067 self.failIfIn("<html>", res)
4068 self.failUnlessIn("Thank you for your report!", res)
4069 d.addCallback(_done)
4072 def test_static(self):
4073 webdir = os.path.join(self.staticdir, "subdir")
4074 fileutil.make_dirs(webdir)
4075 f = open(os.path.join(webdir, "hello.txt"), "wb")
4079 d = self.GET("/static/subdir/hello.txt")
4081 self.failUnlessReallyEqual(res, "hello")
4082 d.addCallback(_check)
4086 class IntroducerWeb(unittest.TestCase):
4091 d = defer.succeed(None)
4093 d.addCallback(lambda ign: self.node.stopService())
4094 d.addCallback(flushEventualQueue)
4097 def test_welcome(self):
4098 basedir = "web.IntroducerWeb.test_welcome"
4100 fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4101 self.node = IntroducerNode(basedir)
4102 self.ws = self.node.getServiceNamed("webish")
4104 d = fireEventually(None)
4105 d.addCallback(lambda ign: self.node.startService())
4106 d.addCallback(lambda ign: self.node.when_tub_ready())
4108 d.addCallback(lambda ign: self.GET("/"))
4110 self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4111 self.failUnlessIn(FAVICON_MARKUP, res)
4112 d.addCallback(_check)
4115 def GET(self, urlpath, followRedirect=False, return_response=False,
4117 # if return_response=True, this fires with (data, statuscode,
4118 # respheaders) instead of just data.
4119 assert not isinstance(urlpath, unicode)
4120 url = self.ws.getURL().rstrip('/') + urlpath
4121 factory = HTTPClientGETFactory(url, method="GET",
4122 followRedirect=followRedirect, **kwargs)
4123 reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4124 d = factory.deferred
4125 def _got_data(data):
4126 return (data, factory.status, factory.response_headers)
4128 d.addCallback(_got_data)
4129 return factory.deferred
4132 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4133 def test_load_file(self):
4134 # This will raise an exception unless a well-formed XML file is found under that name.
4135 common.getxmlfile('directory.xhtml').load()
4137 def test_parse_replace_arg(self):
4138 self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4139 self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4140 self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4142 self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4143 common.parse_replace_arg, "only_fles")
4145 def test_abbreviate_time(self):
4146 self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4147 self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4148 self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4149 self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4150 self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4151 self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4153 def test_compute_rate(self):
4154 self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4155 self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4156 self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4157 self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4158 self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4159 self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4160 self.shouldFail(AssertionError, "test_compute_rate", "",
4161 common.compute_rate, -100, 10)
4162 self.shouldFail(AssertionError, "test_compute_rate", "",
4163 common.compute_rate, 100, -10)
4166 rate = common.compute_rate(10*1000*1000, 1)
4167 self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4169 def test_abbreviate_rate(self):
4170 self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4171 self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4172 self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4173 self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4175 def test_abbreviate_size(self):
4176 self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4177 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4178 self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4179 self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4180 self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4182 def test_plural(self):
4184 return "%d second%s" % (s, status.plural(s))
4185 self.failUnlessReallyEqual(convert(0), "0 seconds")
4186 self.failUnlessReallyEqual(convert(1), "1 second")
4187 self.failUnlessReallyEqual(convert(2), "2 seconds")
4189 return "has share%s: %s" % (status.plural(s), ",".join(s))
4190 self.failUnlessReallyEqual(convert2([]), "has shares: ")
4191 self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4192 self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4195 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4197 def CHECK(self, ign, which, args, clientnum=0):
4198 fileurl = self.fileurls[which]
4199 url = fileurl + "?" + args
4200 return self.GET(url, method="POST", clientnum=clientnum)
4202 def test_filecheck(self):
4203 self.basedir = "web/Grid/filecheck"
4205 c0 = self.g.clients[0]
4208 d = c0.upload(upload.Data(DATA, convergence=""))
4209 def _stash_uri(ur, which):
4210 self.uris[which] = ur.get_uri()
4211 d.addCallback(_stash_uri, "good")
4212 d.addCallback(lambda ign:
4213 c0.upload(upload.Data(DATA+"1", convergence="")))
4214 d.addCallback(_stash_uri, "sick")
4215 d.addCallback(lambda ign:
4216 c0.upload(upload.Data(DATA+"2", convergence="")))
4217 d.addCallback(_stash_uri, "dead")
4218 def _stash_mutable_uri(n, which):
4219 self.uris[which] = n.get_uri()
4220 assert isinstance(self.uris[which], str)
4221 d.addCallback(lambda ign:
4222 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4223 d.addCallback(_stash_mutable_uri, "corrupt")
4224 d.addCallback(lambda ign:
4225 c0.upload(upload.Data("literal", convergence="")))
4226 d.addCallback(_stash_uri, "small")
4227 d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4228 d.addCallback(_stash_mutable_uri, "smalldir")
4230 def _compute_fileurls(ignored):
4232 for which in self.uris:
4233 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4234 d.addCallback(_compute_fileurls)
4236 def _clobber_shares(ignored):
4237 good_shares = self.find_uri_shares(self.uris["good"])
4238 self.failUnlessReallyEqual(len(good_shares), 10)
4239 sick_shares = self.find_uri_shares(self.uris["sick"])
4240 os.unlink(sick_shares[0][2])
4241 dead_shares = self.find_uri_shares(self.uris["dead"])
4242 for i in range(1, 10):
4243 os.unlink(dead_shares[i][2])
4244 c_shares = self.find_uri_shares(self.uris["corrupt"])
4245 cso = CorruptShareOptions()
4246 cso.stdout = StringIO()
4247 cso.parseOptions([c_shares[0][2]])
4249 d.addCallback(_clobber_shares)
4251 d.addCallback(self.CHECK, "good", "t=check")
4252 def _got_html_good(res):
4253 self.failUnlessIn("Healthy", res)
4254 self.failIfIn("Not Healthy", res)
4255 self.failUnlessIn(FAVICON_MARKUP, res)
4256 d.addCallback(_got_html_good)
4257 d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4258 def _got_html_good_return_to(res):
4259 self.failUnlessIn("Healthy", res)
4260 self.failIfIn("Not Healthy", res)
4261 self.failUnlessIn('<a href="somewhere">Return to file', res)
4262 d.addCallback(_got_html_good_return_to)
4263 d.addCallback(self.CHECK, "good", "t=check&output=json")
4264 def _got_json_good(res):
4265 r = simplejson.loads(res)
4266 self.failUnlessEqual(r["summary"], "Healthy")
4267 self.failUnless(r["results"]["healthy"])
4268 self.failIf(r["results"]["needs-rebalancing"])
4269 self.failUnless(r["results"]["recoverable"])
4270 d.addCallback(_got_json_good)
4272 d.addCallback(self.CHECK, "small", "t=check")
4273 def _got_html_small(res):
4274 self.failUnlessIn("Literal files are always healthy", res)
4275 self.failIfIn("Not Healthy", res)
4276 d.addCallback(_got_html_small)
4277 d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4278 def _got_html_small_return_to(res):
4279 self.failUnlessIn("Literal files are always healthy", res)
4280 self.failIfIn("Not Healthy", res)
4281 self.failUnlessIn('<a href="somewhere">Return to file', res)
4282 d.addCallback(_got_html_small_return_to)
4283 d.addCallback(self.CHECK, "small", "t=check&output=json")
4284 def _got_json_small(res):
4285 r = simplejson.loads(res)
4286 self.failUnlessEqual(r["storage-index"], "")
4287 self.failUnless(r["results"]["healthy"])
4288 d.addCallback(_got_json_small)
4290 d.addCallback(self.CHECK, "smalldir", "t=check")
4291 def _got_html_smalldir(res):
4292 self.failUnlessIn("Literal files are always healthy", res)
4293 self.failIfIn("Not Healthy", res)
4294 d.addCallback(_got_html_smalldir)
4295 d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4296 def _got_json_smalldir(res):
4297 r = simplejson.loads(res)
4298 self.failUnlessEqual(r["storage-index"], "")
4299 self.failUnless(r["results"]["healthy"])
4300 d.addCallback(_got_json_smalldir)
4302 d.addCallback(self.CHECK, "sick", "t=check")
4303 def _got_html_sick(res):
4304 self.failUnlessIn("Not Healthy", res)
4305 d.addCallback(_got_html_sick)
4306 d.addCallback(self.CHECK, "sick", "t=check&output=json")
4307 def _got_json_sick(res):
4308 r = simplejson.loads(res)
4309 self.failUnlessEqual(r["summary"],
4310 "Not Healthy: 9 shares (enc 3-of-10)")
4311 self.failIf(r["results"]["healthy"])
4312 self.failIf(r["results"]["needs-rebalancing"])
4313 self.failUnless(r["results"]["recoverable"])
4314 d.addCallback(_got_json_sick)
4316 d.addCallback(self.CHECK, "dead", "t=check")
4317 def _got_html_dead(res):
4318 self.failUnlessIn("Not Healthy", res)
4319 d.addCallback(_got_html_dead)
4320 d.addCallback(self.CHECK, "dead", "t=check&output=json")
4321 def _got_json_dead(res):
4322 r = simplejson.loads(res)
4323 self.failUnlessEqual(r["summary"],
4324 "Not Healthy: 1 shares (enc 3-of-10)")
4325 self.failIf(r["results"]["healthy"])
4326 self.failIf(r["results"]["needs-rebalancing"])
4327 self.failIf(r["results"]["recoverable"])
4328 d.addCallback(_got_json_dead)
4330 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4331 def _got_html_corrupt(res):
4332 self.failUnlessIn("Not Healthy! : Unhealthy", res)
4333 d.addCallback(_got_html_corrupt)
4334 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4335 def _got_json_corrupt(res):
4336 r = simplejson.loads(res)
4337 self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4338 self.failIf(r["results"]["healthy"])
4339 self.failUnless(r["results"]["recoverable"])
4340 self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4341 self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4342 d.addCallback(_got_json_corrupt)
4344 d.addErrback(self.explain_web_error)
4347 def test_repair_html(self):
4348 self.basedir = "web/Grid/repair_html"
4350 c0 = self.g.clients[0]
4353 d = c0.upload(upload.Data(DATA, convergence=""))
4354 def _stash_uri(ur, which):
4355 self.uris[which] = ur.get_uri()
4356 d.addCallback(_stash_uri, "good")
4357 d.addCallback(lambda ign:
4358 c0.upload(upload.Data(DATA+"1", convergence="")))
4359 d.addCallback(_stash_uri, "sick")
4360 d.addCallback(lambda ign:
4361 c0.upload(upload.Data(DATA+"2", convergence="")))
4362 d.addCallback(_stash_uri, "dead")
4363 def _stash_mutable_uri(n, which):
4364 self.uris[which] = n.get_uri()
4365 assert isinstance(self.uris[which], str)
4366 d.addCallback(lambda ign:
4367 c0.create_mutable_file(publish.MutableData(DATA+"3")))
4368 d.addCallback(_stash_mutable_uri, "corrupt")
4370 def _compute_fileurls(ignored):
4372 for which in self.uris:
4373 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4374 d.addCallback(_compute_fileurls)
4376 def _clobber_shares(ignored):
4377 good_shares = self.find_uri_shares(self.uris["good"])
4378 self.failUnlessReallyEqual(len(good_shares), 10)
4379 sick_shares = self.find_uri_shares(self.uris["sick"])
4380 os.unlink(sick_shares[0][2])
4381 dead_shares = self.find_uri_shares(self.uris["dead"])
4382 for i in range(1, 10):
4383 os.unlink(dead_shares[i][2])
4384 c_shares = self.find_uri_shares(self.uris["corrupt"])
4385 cso = CorruptShareOptions()
4386 cso.stdout = StringIO()
4387 cso.parseOptions([c_shares[0][2]])
4389 d.addCallback(_clobber_shares)
4391 d.addCallback(self.CHECK, "good", "t=check&repair=true")
4392 def _got_html_good(res):
4393 self.failUnlessIn("Healthy", res)
4394 self.failIfIn("Not Healthy", res)
4395 self.failUnlessIn("No repair necessary", res)
4396 self.failUnlessIn(FAVICON_MARKUP, res)
4397 d.addCallback(_got_html_good)
4399 d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4400 def _got_html_sick(res):
4401 self.failUnlessIn("Healthy : healthy", res)
4402 self.failIfIn("Not Healthy", res)
4403 self.failUnlessIn("Repair successful", res)
4404 d.addCallback(_got_html_sick)
4406 # repair of a dead file will fail, of course, but it isn't yet
4407 # clear how this should be reported. Right now it shows up as
4410 #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4411 #def _got_html_dead(res):
4413 # self.failUnlessIn("Healthy : healthy", res)
4414 # self.failIfIn("Not Healthy", res)
4415 # self.failUnlessIn("No repair necessary", res)
4416 #d.addCallback(_got_html_dead)
4418 d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4419 def _got_html_corrupt(res):
4420 self.failUnlessIn("Healthy : Healthy", res)
4421 self.failIfIn("Not Healthy", res)
4422 self.failUnlessIn("Repair successful", res)
4423 d.addCallback(_got_html_corrupt)
4425 d.addErrback(self.explain_web_error)
4428 def test_repair_json(self):
4429 self.basedir = "web/Grid/repair_json"
4431 c0 = self.g.clients[0]
4434 d = c0.upload(upload.Data(DATA+"1", convergence=""))
4435 def _stash_uri(ur, which):
4436 self.uris[which] = ur.get_uri()
4437 d.addCallback(_stash_uri, "sick")
4439 def _compute_fileurls(ignored):
4441 for which in self.uris:
4442 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4443 d.addCallback(_compute_fileurls)
4445 def _clobber_shares(ignored):
4446 sick_shares = self.find_uri_shares(self.uris["sick"])
4447 os.unlink(sick_shares[0][2])
4448 d.addCallback(_clobber_shares)
4450 d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4451 def _got_json_sick(res):
4452 r = simplejson.loads(res)
4453 self.failUnlessReallyEqual(r["repair-attempted"], True)
4454 self.failUnlessReallyEqual(r["repair-successful"], True)
4455 self.failUnlessEqual(r["pre-repair-results"]["summary"],
4456 "Not Healthy: 9 shares (enc 3-of-10)")
4457 self.failIf(r["pre-repair-results"]["results"]["healthy"])
4458 self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4459 self.failUnless(r["post-repair-results"]["results"]["healthy"])
4460 d.addCallback(_got_json_sick)
4462 d.addErrback(self.explain_web_error)
4465 def test_unknown(self, immutable=False):
4466 self.basedir = "web/Grid/unknown"
4468 self.basedir = "web/Grid/unknown-immutable"
4471 c0 = self.g.clients[0]
4475 # the future cap format may contain slashes, which must be tolerated
4476 expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4480 name = u"future-imm"
4481 future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4482 d = c0.create_immutable_dirnode({name: (future_node, {})})
4485 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4486 d = c0.create_dirnode()
4488 def _stash_root_and_create_file(n):
4490 self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4491 self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4493 return self.rootnode.set_node(name, future_node)
4494 d.addCallback(_stash_root_and_create_file)
4496 # make sure directory listing tolerates unknown nodes
4497 d.addCallback(lambda ign: self.GET(self.rooturl))
4498 def _check_directory_html(res, expected_type_suffix):
4499 pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4500 '<td>%s</td>' % (expected_type_suffix, str(name)),
4502 self.failUnless(re.search(pattern, res), res)
4503 # find the More Info link for name, should be relative
4504 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4505 info_url = mo.group(1)
4506 self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4508 d.addCallback(_check_directory_html, "-IMM")
4510 d.addCallback(_check_directory_html, "")
4512 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4513 def _check_directory_json(res, expect_rw_uri):
4514 data = simplejson.loads(res)
4515 self.failUnlessEqual(data[0], "dirnode")
4516 f = data[1]["children"][name]
4517 self.failUnlessEqual(f[0], "unknown")
4519 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4521 self.failIfIn("rw_uri", f[1])
4523 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4525 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4526 self.failUnlessIn("metadata", f[1])
4527 d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4529 def _check_info(res, expect_rw_uri, expect_ro_uri):
4530 self.failUnlessIn("Object Type: <span>unknown</span>", res)
4532 self.failUnlessIn(unknown_rwcap, res)
4535 self.failUnlessIn(unknown_immcap, res)
4537 self.failUnlessIn(unknown_rocap, res)
4539 self.failIfIn(unknown_rocap, res)
4540 self.failIfIn("Raw data as", res)
4541 self.failIfIn("Directory writecap", res)
4542 self.failIfIn("Checker Operations", res)
4543 self.failIfIn("Mutable File Operations", res)
4544 self.failIfIn("Directory Operations", res)
4546 # FIXME: these should have expect_rw_uri=not immutable; I don't know
4547 # why they fail. Possibly related to ticket #922.
4549 d.addCallback(lambda ign: self.GET(expected_info_url))
4550 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4551 d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4552 d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4554 def _check_json(res, expect_rw_uri):
4555 data = simplejson.loads(res)
4556 self.failUnlessEqual(data[0], "unknown")
4558 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4560 self.failIfIn("rw_uri", data[1])
4563 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4564 self.failUnlessReallyEqual(data[1]["mutable"], False)
4566 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4567 self.failUnlessReallyEqual(data[1]["mutable"], True)
4569 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4570 self.failIfIn("mutable", data[1])
4572 # TODO: check metadata contents
4573 self.failUnlessIn("metadata", data[1])
4575 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4576 d.addCallback(_check_json, expect_rw_uri=not immutable)
4578 # and make sure that a read-only version of the directory can be
4579 # rendered too. This version will not have unknown_rwcap, whether
4580 # or not future_node was immutable.
4581 d.addCallback(lambda ign: self.GET(self.rourl))
4583 d.addCallback(_check_directory_html, "-IMM")
4585 d.addCallback(_check_directory_html, "-RO")
4587 d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4588 d.addCallback(_check_directory_json, expect_rw_uri=False)
4590 d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4591 d.addCallback(_check_json, expect_rw_uri=False)
4593 # TODO: check that getting t=info from the Info link in the ro directory
4594 # works, and does not include the writecap URI.
4597 def test_immutable_unknown(self):
4598 return self.test_unknown(immutable=True)
4600 def test_mutant_dirnodes_are_omitted(self):
4601 self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4604 c = self.g.clients[0]
4609 lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4610 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4611 mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4613 # This method tests mainly dirnode, but we'd have to duplicate code in order to
4614 # test the dirnode and web layers separately.
4616 # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4617 # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4618 # When the directory is read, the mutants should be silently disposed of, leaving
4619 # their lonely sibling.
4620 # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4621 # because immutable directories don't have a writecap and therefore that field
4622 # isn't (and can't be) decrypted.
4623 # TODO: The field still exists in the netstring. Technically we should check what
4624 # happens if something is put there (_unpack_contents should raise ValueError),
4625 # but that can wait.
4627 lonely_child = nm.create_from_cap(lonely_uri)
4628 mutant_ro_child = nm.create_from_cap(mut_read_uri)
4629 mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4631 def _by_hook_or_by_crook():
4633 for n in [mutant_ro_child, mutant_write_in_ro_child]:
4634 n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4636 mutant_write_in_ro_child.get_write_uri = lambda: None
4637 mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4639 kids = {u"lonely": (lonely_child, {}),
4640 u"ro": (mutant_ro_child, {}),
4641 u"write-in-ro": (mutant_write_in_ro_child, {}),
4643 d = c.create_immutable_dirnode(kids)
4646 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4647 self.failIf(dn.is_mutable())
4648 self.failUnless(dn.is_readonly())
4649 # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4650 self.failIf(hasattr(dn._node, 'get_writekey'))
4652 self.failUnlessIn("RO-IMM", rep)
4654 self.failUnlessIn("CHK", cap.to_string())
4657 self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4658 return download_to_data(dn._node)
4659 d.addCallback(_created)
4661 def _check_data(data):
4662 # Decode the netstring representation of the directory to check that all children
4663 # are present. This is a bit of an abstraction violation, but there's not really
4664 # any other way to do it given that the real DirectoryNode._unpack_contents would
4665 # strip the mutant children out (which is what we're trying to test, later).
4668 while position < len(data):
4669 entries, position = split_netstring(data, 1, position)
4671 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4672 name = name_utf8.decode("utf-8")
4673 self.failUnlessEqual(rwcapdata, "")
4674 self.failUnlessIn(name, kids)
4675 (expected_child, ign) = kids[name]
4676 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4679 self.failUnlessReallyEqual(numkids, 3)
4680 return self.rootnode.list()
4681 d.addCallback(_check_data)
4683 # Now when we use the real directory listing code, the mutants should be absent.
4684 def _check_kids(children):
4685 self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4686 lonely_node, lonely_metadata = children[u"lonely"]
4688 self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4689 self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4690 d.addCallback(_check_kids)
4692 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4693 d.addCallback(lambda n: n.list())
4694 d.addCallback(_check_kids) # again with dirnode recreated from cap
4696 # Make sure the lonely child can be listed in HTML...
4697 d.addCallback(lambda ign: self.GET(self.rooturl))
4698 def _check_html(res):
4699 self.failIfIn("URI:SSK", res)
4700 get_lonely = "".join([r'<td>FILE</td>',
4702 r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4704 r'\s+<td align="right">%d</td>' % len("one"),
4706 self.failUnless(re.search(get_lonely, res), res)
4708 # find the More Info link for name, should be relative
4709 mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4710 info_url = mo.group(1)
4711 self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4712 d.addCallback(_check_html)
4715 d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4716 def _check_json(res):
4717 data = simplejson.loads(res)
4718 self.failUnlessEqual(data[0], "dirnode")
4719 listed_children = data[1]["children"]
4720 self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4721 ll_type, ll_data = listed_children[u"lonely"]
4722 self.failUnlessEqual(ll_type, "filenode")
4723 self.failIfIn("rw_uri", ll_data)
4724 self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4725 d.addCallback(_check_json)
4728 def test_deep_check(self):
4729 self.basedir = "web/Grid/deep_check"
4731 c0 = self.g.clients[0]
4735 d = c0.create_dirnode()
4736 def _stash_root_and_create_file(n):
4738 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4739 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4740 d.addCallback(_stash_root_and_create_file)
4741 def _stash_uri(fn, which):
4742 self.uris[which] = fn.get_uri()
4744 d.addCallback(_stash_uri, "good")
4745 d.addCallback(lambda ign:
4746 self.rootnode.add_file(u"small",
4747 upload.Data("literal",
4749 d.addCallback(_stash_uri, "small")
4750 d.addCallback(lambda ign:
4751 self.rootnode.add_file(u"sick",
4752 upload.Data(DATA+"1",
4754 d.addCallback(_stash_uri, "sick")
4756 # this tests that deep-check and stream-manifest will ignore
4757 # UnknownNode instances. Hopefully this will also cover deep-stats.
4758 future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4759 d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4761 def _clobber_shares(ignored):
4762 self.delete_shares_numbered(self.uris["sick"], [0,1])
4763 d.addCallback(_clobber_shares)
4771 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4774 units = [simplejson.loads(line)
4775 for line in res.splitlines()
4778 print "response is:", res
4779 print "undecodeable line was '%s'" % line
4781 self.failUnlessReallyEqual(len(units), 5+1)
4782 # should be parent-first
4784 self.failUnlessEqual(u0["path"], [])
4785 self.failUnlessEqual(u0["type"], "directory")
4786 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4787 u0cr = u0["check-results"]
4788 self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4790 ugood = [u for u in units
4791 if u["type"] == "file" and u["path"] == [u"good"]][0]
4792 self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4793 ugoodcr = ugood["check-results"]
4794 self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4797 self.failUnlessEqual(stats["type"], "stats")
4799 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4800 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4801 self.failUnlessReallyEqual(s["count-directories"], 1)
4802 self.failUnlessReallyEqual(s["count-unknown"], 1)
4803 d.addCallback(_done)
4805 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4806 def _check_manifest(res):
4807 self.failUnless(res.endswith("\n"))
4808 units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4809 self.failUnlessReallyEqual(len(units), 5+1)
4810 self.failUnlessEqual(units[-1]["type"], "stats")
4812 self.failUnlessEqual(first["path"], [])
4813 self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4814 self.failUnlessEqual(first["type"], "directory")
4815 stats = units[-1]["stats"]
4816 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4817 self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4818 self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4819 self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4820 self.failUnlessReallyEqual(stats["count-unknown"], 1)
4821 d.addCallback(_check_manifest)
4823 # now add root/subdir and root/subdir/grandchild, then make subdir
4824 # unrecoverable, then see what happens
4826 d.addCallback(lambda ign:
4827 self.rootnode.create_subdirectory(u"subdir"))
4828 d.addCallback(_stash_uri, "subdir")
4829 d.addCallback(lambda subdir_node:
4830 subdir_node.add_file(u"grandchild",
4831 upload.Data(DATA+"2",
4833 d.addCallback(_stash_uri, "grandchild")
4835 d.addCallback(lambda ign:
4836 self.delete_shares_numbered(self.uris["subdir"],
4844 # root/subdir [unrecoverable]
4845 # root/subdir/grandchild
4847 # how should a streaming-JSON API indicate fatal error?
4848 # answer: emit ERROR: instead of a JSON string
4850 d.addCallback(self.CHECK, "root", "t=stream-manifest")
4851 def _check_broken_manifest(res):
4852 lines = res.splitlines()
4854 for (i,line) in enumerate(lines)
4855 if line.startswith("ERROR:")]
4857 self.fail("no ERROR: in output: %s" % (res,))
4858 first_error = error_lines[0]
4859 error_line = lines[first_error]
4860 error_msg = lines[first_error+1:]
4861 error_msg_s = "\n".join(error_msg) + "\n"
4862 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4864 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4865 units = [simplejson.loads(line) for line in lines[:first_error]]
4866 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4867 last_unit = units[-1]
4868 self.failUnlessEqual(last_unit["path"], ["subdir"])
4869 d.addCallback(_check_broken_manifest)
4871 d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4872 def _check_broken_deepcheck(res):
4873 lines = res.splitlines()
4875 for (i,line) in enumerate(lines)
4876 if line.startswith("ERROR:")]
4878 self.fail("no ERROR: in output: %s" % (res,))
4879 first_error = error_lines[0]
4880 error_line = lines[first_error]
4881 error_msg = lines[first_error+1:]
4882 error_msg_s = "\n".join(error_msg) + "\n"
4883 self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4885 self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4886 units = [simplejson.loads(line) for line in lines[:first_error]]
4887 self.failUnlessReallyEqual(len(units), 6) # includes subdir
4888 last_unit = units[-1]
4889 self.failUnlessEqual(last_unit["path"], ["subdir"])
4890 r = last_unit["check-results"]["results"]
4891 self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4892 self.failUnlessReallyEqual(r["count-shares-good"], 1)
4893 self.failUnlessReallyEqual(r["recoverable"], False)
4894 d.addCallback(_check_broken_deepcheck)
4896 d.addErrback(self.explain_web_error)
4899 def test_deep_check_and_repair(self):
4900 self.basedir = "web/Grid/deep_check_and_repair"
4902 c0 = self.g.clients[0]
4906 d = c0.create_dirnode()
4907 def _stash_root_and_create_file(n):
4909 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4910 return n.add_file(u"good", upload.Data(DATA, convergence=""))
4911 d.addCallback(_stash_root_and_create_file)
4912 def _stash_uri(fn, which):
4913 self.uris[which] = fn.get_uri()
4914 d.addCallback(_stash_uri, "good")
4915 d.addCallback(lambda ign:
4916 self.rootnode.add_file(u"small",
4917 upload.Data("literal",
4919 d.addCallback(_stash_uri, "small")
4920 d.addCallback(lambda ign:
4921 self.rootnode.add_file(u"sick",
4922 upload.Data(DATA+"1",
4924 d.addCallback(_stash_uri, "sick")
4925 #d.addCallback(lambda ign:
4926 # self.rootnode.add_file(u"dead",
4927 # upload.Data(DATA+"2",
4929 #d.addCallback(_stash_uri, "dead")
4931 #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4932 #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4933 #d.addCallback(_stash_uri, "corrupt")
4935 def _clobber_shares(ignored):
4936 good_shares = self.find_uri_shares(self.uris["good"])
4937 self.failUnlessReallyEqual(len(good_shares), 10)
4938 sick_shares = self.find_uri_shares(self.uris["sick"])
4939 os.unlink(sick_shares[0][2])
4940 #dead_shares = self.find_uri_shares(self.uris["dead"])
4941 #for i in range(1, 10):
4942 # os.unlink(dead_shares[i][2])
4944 #c_shares = self.find_uri_shares(self.uris["corrupt"])
4945 #cso = CorruptShareOptions()
4946 #cso.stdout = StringIO()
4947 #cso.parseOptions([c_shares[0][2]])
4949 d.addCallback(_clobber_shares)
4952 # root/good CHK, 10 shares
4954 # root/sick CHK, 9 shares
4956 d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4958 units = [simplejson.loads(line)
4959 for line in res.splitlines()
4961 self.failUnlessReallyEqual(len(units), 4+1)
4962 # should be parent-first
4964 self.failUnlessEqual(u0["path"], [])
4965 self.failUnlessEqual(u0["type"], "directory")
4966 self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4967 u0crr = u0["check-and-repair-results"]
4968 self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4969 self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4971 ugood = [u for u in units
4972 if u["type"] == "file" and u["path"] == [u"good"]][0]
4973 self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4974 ugoodcrr = ugood["check-and-repair-results"]
4975 self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4976 self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4978 usick = [u for u in units
4979 if u["type"] == "file" and u["path"] == [u"sick"]][0]
4980 self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4981 usickcrr = usick["check-and-repair-results"]
4982 self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4983 self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4984 self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4985 self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4988 self.failUnlessEqual(stats["type"], "stats")
4990 self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4991 self.failUnlessReallyEqual(s["count-literal-files"], 1)
4992 self.failUnlessReallyEqual(s["count-directories"], 1)
4993 d.addCallback(_done)
4995 d.addErrback(self.explain_web_error)
4998 def _count_leases(self, ignored, which):
4999 u = self.uris[which]
5000 shares = self.find_uri_shares(u)
5002 for shnum, serverid, fn in shares:
5003 sf = get_share_file(fn)
5004 num_leases = len(list(sf.get_leases()))
5005 lease_counts.append( (fn, num_leases) )
5008 def _assert_leasecount(self, lease_counts, expected):
5009 for (fn, num_leases) in lease_counts:
5010 if num_leases != expected:
5011 self.fail("expected %d leases, have %d, on %s" %
5012 (expected, num_leases, fn))
5014 def test_add_lease(self):
5015 self.basedir = "web/Grid/add_lease"
5016 self.set_up_grid(num_clients=2)
5017 c0 = self.g.clients[0]
5020 d = c0.upload(upload.Data(DATA, convergence=""))
5021 def _stash_uri(ur, which):
5022 self.uris[which] = ur.get_uri()
5023 d.addCallback(_stash_uri, "one")
5024 d.addCallback(lambda ign:
5025 c0.upload(upload.Data(DATA+"1", convergence="")))
5026 d.addCallback(_stash_uri, "two")
5027 def _stash_mutable_uri(n, which):
5028 self.uris[which] = n.get_uri()
5029 assert isinstance(self.uris[which], str)
5030 d.addCallback(lambda ign:
5031 c0.create_mutable_file(publish.MutableData(DATA+"2")))
5032 d.addCallback(_stash_mutable_uri, "mutable")
5034 def _compute_fileurls(ignored):
5036 for which in self.uris:
5037 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5038 d.addCallback(_compute_fileurls)
5040 d.addCallback(self._count_leases, "one")
5041 d.addCallback(self._assert_leasecount, 1)
5042 d.addCallback(self._count_leases, "two")
5043 d.addCallback(self._assert_leasecount, 1)
5044 d.addCallback(self._count_leases, "mutable")
5045 d.addCallback(self._assert_leasecount, 1)
5047 d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5048 def _got_html_good(res):
5049 self.failUnlessIn("Healthy", res)
5050 self.failIfIn("Not Healthy", res)
5051 d.addCallback(_got_html_good)
5053 d.addCallback(self._count_leases, "one")
5054 d.addCallback(self._assert_leasecount, 1)
5055 d.addCallback(self._count_leases, "two")
5056 d.addCallback(self._assert_leasecount, 1)
5057 d.addCallback(self._count_leases, "mutable")
5058 d.addCallback(self._assert_leasecount, 1)
5060 # this CHECK uses the original client, which uses the same
5061 # lease-secrets, so it will just renew the original lease
5062 d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5063 d.addCallback(_got_html_good)
5065 d.addCallback(self._count_leases, "one")
5066 d.addCallback(self._assert_leasecount, 1)
5067 d.addCallback(self._count_leases, "two")
5068 d.addCallback(self._assert_leasecount, 1)
5069 d.addCallback(self._count_leases, "mutable")
5070 d.addCallback(self._assert_leasecount, 1)
5072 # this CHECK uses an alternate client, which adds a second lease
5073 d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5074 d.addCallback(_got_html_good)
5076 d.addCallback(self._count_leases, "one")
5077 d.addCallback(self._assert_leasecount, 2)
5078 d.addCallback(self._count_leases, "two")
5079 d.addCallback(self._assert_leasecount, 1)
5080 d.addCallback(self._count_leases, "mutable")
5081 d.addCallback(self._assert_leasecount, 1)
5083 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5084 d.addCallback(_got_html_good)
5086 d.addCallback(self._count_leases, "one")
5087 d.addCallback(self._assert_leasecount, 2)
5088 d.addCallback(self._count_leases, "two")
5089 d.addCallback(self._assert_leasecount, 1)
5090 d.addCallback(self._count_leases, "mutable")
5091 d.addCallback(self._assert_leasecount, 1)
5093 d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5095 d.addCallback(_got_html_good)
5097 d.addCallback(self._count_leases, "one")
5098 d.addCallback(self._assert_leasecount, 2)
5099 d.addCallback(self._count_leases, "two")
5100 d.addCallback(self._assert_leasecount, 1)
5101 d.addCallback(self._count_leases, "mutable")
5102 d.addCallback(self._assert_leasecount, 2)
5104 d.addErrback(self.explain_web_error)
5107 def test_deep_add_lease(self):
5108 self.basedir = "web/Grid/deep_add_lease"
5109 self.set_up_grid(num_clients=2)
5110 c0 = self.g.clients[0]
5114 d = c0.create_dirnode()
5115 def _stash_root_and_create_file(n):
5117 self.uris["root"] = n.get_uri()
5118 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5119 return n.add_file(u"one", upload.Data(DATA, convergence=""))
5120 d.addCallback(_stash_root_and_create_file)
5121 def _stash_uri(fn, which):
5122 self.uris[which] = fn.get_uri()
5123 d.addCallback(_stash_uri, "one")
5124 d.addCallback(lambda ign:
5125 self.rootnode.add_file(u"small",
5126 upload.Data("literal",
5128 d.addCallback(_stash_uri, "small")
5130 d.addCallback(lambda ign:
5131 c0.create_mutable_file(publish.MutableData("mutable")))
5132 d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5133 d.addCallback(_stash_uri, "mutable")
5135 d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5137 units = [simplejson.loads(line)
5138 for line in res.splitlines()
5140 # root, one, small, mutable, stats
5141 self.failUnlessReallyEqual(len(units), 4+1)
5142 d.addCallback(_done)
5144 d.addCallback(self._count_leases, "root")
5145 d.addCallback(self._assert_leasecount, 1)
5146 d.addCallback(self._count_leases, "one")
5147 d.addCallback(self._assert_leasecount, 1)
5148 d.addCallback(self._count_leases, "mutable")
5149 d.addCallback(self._assert_leasecount, 1)
5151 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5152 d.addCallback(_done)
5154 d.addCallback(self._count_leases, "root")
5155 d.addCallback(self._assert_leasecount, 1)
5156 d.addCallback(self._count_leases, "one")
5157 d.addCallback(self._assert_leasecount, 1)
5158 d.addCallback(self._count_leases, "mutable")
5159 d.addCallback(self._assert_leasecount, 1)
5161 d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5163 d.addCallback(_done)
5165 d.addCallback(self._count_leases, "root")
5166 d.addCallback(self._assert_leasecount, 2)
5167 d.addCallback(self._count_leases, "one")
5168 d.addCallback(self._assert_leasecount, 2)
5169 d.addCallback(self._count_leases, "mutable")
5170 d.addCallback(self._assert_leasecount, 2)
5172 d.addErrback(self.explain_web_error)
5176 def test_exceptions(self):
5177 self.basedir = "web/Grid/exceptions"
5178 self.set_up_grid(num_clients=1, num_servers=2)
5179 c0 = self.g.clients[0]
5180 c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5183 d = c0.create_dirnode()
5185 self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5186 self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5188 d.addCallback(_stash_root)
5189 d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5191 self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5192 self.delete_shares_numbered(ur.get_uri(), range(1,10))
5194 u = uri.from_string(ur.get_uri())
5195 u.key = testutil.flip_bit(u.key, 0)
5196 baduri = u.to_string()
5197 self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5198 d.addCallback(_stash_bad)
5199 d.addCallback(lambda ign: c0.create_dirnode())
5200 def _mangle_dirnode_1share(n):
5202 url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5203 self.fileurls["dir-1share-json"] = url + "?t=json"
5204 self.delete_shares_numbered(u, range(1,10))
5205 d.addCallback(_mangle_dirnode_1share)
5206 d.addCallback(lambda ign: c0.create_dirnode())
5207 def _mangle_dirnode_0share(n):
5209 url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5210 self.fileurls["dir-0share-json"] = url + "?t=json"
5211 self.delete_shares_numbered(u, range(0,10))
5212 d.addCallback(_mangle_dirnode_0share)
5214 # NotEnoughSharesError should be reported sensibly, with a
5215 # text/plain explanation of the problem, and perhaps some
5216 # information on which shares *could* be found.
5218 d.addCallback(lambda ignored:
5219 self.shouldHTTPError("GET unrecoverable",
5220 410, "Gone", "NoSharesError",
5221 self.GET, self.fileurls["0shares"]))
5222 def _check_zero_shares(body):
5223 self.failIfIn("<html>", body)
5224 body = " ".join(body.strip().split())
5225 exp = ("NoSharesError: no shares could be found. "
5226 "Zero shares usually indicates a corrupt URI, or that "
5227 "no servers were connected, but it might also indicate "
5228 "severe corruption. You should perform a filecheck on "
5229 "this object to learn more. The full error message is: "
5230 "no shares (need 3). Last failure: None")
5231 self.failUnlessReallyEqual(exp, body)
5232 d.addCallback(_check_zero_shares)
5235 d.addCallback(lambda ignored:
5236 self.shouldHTTPError("GET 1share",
5237 410, "Gone", "NotEnoughSharesError",
5238 self.GET, self.fileurls["1share"]))
5239 def _check_one_share(body):
5240 self.failIfIn("<html>", body)
5241 body = " ".join(body.strip().split())
5242 msgbase = ("NotEnoughSharesError: This indicates that some "
5243 "servers were unavailable, or that shares have been "
5244 "lost to server departure, hard drive failure, or disk "
5245 "corruption. You should perform a filecheck on "
5246 "this object to learn more. The full error message is:"
5248 msg1 = msgbase + (" ran out of shares:"
5251 " overdue= unused= need 3. Last failure: None")
5252 msg2 = msgbase + (" ran out of shares:"
5254 " pending=Share(sh0-on-xgru5)"
5255 " overdue= unused= need 3. Last failure: None")
5256 self.failUnless(body == msg1 or body == msg2, body)
5257 d.addCallback(_check_one_share)
5259 d.addCallback(lambda ignored:
5260 self.shouldHTTPError("GET imaginary",
5261 404, "Not Found", None,
5262 self.GET, self.fileurls["imaginary"]))
5263 def _missing_child(body):
5264 self.failUnlessIn("No such child: imaginary", body)
5265 d.addCallback(_missing_child)
5267 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5268 def _check_0shares_dir_html(body):
5269 self.failUnlessIn("<html>", body)
5270 # we should see the regular page, but without the child table or
5272 body = " ".join(body.strip().split())
5273 self.failUnlessIn('href="?t=info">More info on this directory',
5275 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5276 "could not be retrieved, because there were insufficient "
5277 "good shares. This might indicate that no servers were "
5278 "connected, insufficient servers were connected, the URI "
5279 "was corrupt, or that shares have been lost due to server "
5280 "departure, hard drive failure, or disk corruption. You "
5281 "should perform a filecheck on this object to learn more.")
5282 self.failUnlessIn(exp, body)
5283 self.failUnlessIn("No upload forms: directory is unreadable", body)
5284 d.addCallback(_check_0shares_dir_html)
5286 d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5287 def _check_1shares_dir_html(body):
5288 # at some point, we'll split UnrecoverableFileError into 0-shares
5289 # and some-shares like we did for immutable files (since there
5290 # are different sorts of advice to offer in each case). For now,
5291 # they present the same way.
5292 self.failUnlessIn("<html>", body)
5293 body = " ".join(body.strip().split())
5294 self.failUnlessIn('href="?t=info">More info on this directory',
5296 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5297 "could not be retrieved, because there were insufficient "
5298 "good shares. This might indicate that no servers were "
5299 "connected, insufficient servers were connected, the URI "
5300 "was corrupt, or that shares have been lost due to server "
5301 "departure, hard drive failure, or disk corruption. You "
5302 "should perform a filecheck on this object to learn more.")
5303 self.failUnlessIn(exp, body)
5304 self.failUnlessIn("No upload forms: directory is unreadable", body)
5305 d.addCallback(_check_1shares_dir_html)
5307 d.addCallback(lambda ignored:
5308 self.shouldHTTPError("GET dir-0share-json",
5309 410, "Gone", "UnrecoverableFileError",
5311 self.fileurls["dir-0share-json"]))
5312 def _check_unrecoverable_file(body):
5313 self.failIfIn("<html>", body)
5314 body = " ".join(body.strip().split())
5315 exp = ("UnrecoverableFileError: the directory (or mutable file) "
5316 "could not be retrieved, because there were insufficient "
5317 "good shares. This might indicate that no servers were "
5318 "connected, insufficient servers were connected, the URI "
5319 "was corrupt, or that shares have been lost due to server "
5320 "departure, hard drive failure, or disk corruption. You "
5321 "should perform a filecheck on this object to learn more.")
5322 self.failUnlessReallyEqual(exp, body)
5323 d.addCallback(_check_unrecoverable_file)
5325 d.addCallback(lambda ignored:
5326 self.shouldHTTPError("GET dir-1share-json",
5327 410, "Gone", "UnrecoverableFileError",
5329 self.fileurls["dir-1share-json"]))
5330 d.addCallback(_check_unrecoverable_file)
5332 d.addCallback(lambda ignored:
5333 self.shouldHTTPError("GET imaginary",
5334 404, "Not Found", None,
5335 self.GET, self.fileurls["imaginary"]))
5337 # attach a webapi child that throws a random error, to test how it
5339 w = c0.getServiceNamed("webish")
5340 w.root.putChild("ERRORBOOM", ErrorBoom())
5342 # "Accept: */*" : should get a text/html stack trace
5343 # "Accept: text/plain" : should get a text/plain stack trace
5344 # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5345 # no Accept header: should get a text/html stack trace
5347 d.addCallback(lambda ignored:
5348 self.shouldHTTPError("GET errorboom_html",
5349 500, "Internal Server Error", None,
5350 self.GET, "ERRORBOOM",
5351 headers={"accept": "*/*"}))
5352 def _internal_error_html1(body):
5353 self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5354 d.addCallback(_internal_error_html1)
5356 d.addCallback(lambda ignored:
5357 self.shouldHTTPError("GET errorboom_text",
5358 500, "Internal Server Error", None,
5359 self.GET, "ERRORBOOM",
5360 headers={"accept": "text/plain"}))
5361 def _internal_error_text2(body):
5362 self.failIfIn("<html>", body)
5363 self.failUnless(body.startswith("Traceback "), body)
5364 d.addCallback(_internal_error_text2)
5366 CLI_accepts = "text/plain, application/octet-stream"
5367 d.addCallback(lambda ignored:
5368 self.shouldHTTPError("GET errorboom_text",
5369 500, "Internal Server Error", None,
5370 self.GET, "ERRORBOOM",
5371 headers={"accept": CLI_accepts}))
5372 def _internal_error_text3(body):
5373 self.failIfIn("<html>", body)
5374 self.failUnless(body.startswith("Traceback "), body)
5375 d.addCallback(_internal_error_text3)
5377 d.addCallback(lambda ignored:
5378 self.shouldHTTPError("GET errorboom_text",
5379 500, "Internal Server Error", None,
5380 self.GET, "ERRORBOOM"))
5381 def _internal_error_html4(body):
5382 self.failUnlessIn("<html>", body)
5383 d.addCallback(_internal_error_html4)
5385 def _flush_errors(res):
5386 # Trial: please ignore the CompletelyUnhandledError in the logs
5387 self.flushLoggedErrors(CompletelyUnhandledError)
5389 d.addBoth(_flush_errors)
5393 def test_blacklist(self):
5394 # download from a blacklisted URI, get an error
5395 self.basedir = "web/Grid/blacklist"
5397 c0 = self.g.clients[0]
5398 c0_basedir = c0.basedir
5399 fn = os.path.join(c0_basedir, "access.blacklist")
5401 DATA = "off-limits " * 50
5403 d = c0.upload(upload.Data(DATA, convergence=""))
5404 def _stash_uri_and_create_dir(ur):
5405 self.uri = ur.get_uri()
5406 self.url = "uri/"+self.uri
5407 u = uri.from_string_filenode(self.uri)
5408 self.si = u.get_storage_index()
5409 childnode = c0.create_node_from_uri(self.uri, None)
5410 return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5411 d.addCallback(_stash_uri_and_create_dir)
5412 def _stash_dir(node):
5413 self.dir_node = node
5414 self.dir_uri = node.get_uri()
5415 self.dir_url = "uri/"+self.dir_uri
5416 d.addCallback(_stash_dir)
5417 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5418 def _check_dir_html(body):
5419 self.failUnlessIn("<html>", body)
5420 self.failUnlessIn("blacklisted.txt</a>", body)
5421 d.addCallback(_check_dir_html)
5422 d.addCallback(lambda ign: self.GET(self.url))
5423 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5425 def _blacklist(ign):
5427 f.write(" # this is a comment\n")
5429 f.write("\n") # also exercise blank lines
5430 f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5432 # clients should be checking the blacklist each time, so we don't
5433 # need to restart the client
5434 d.addCallback(_blacklist)
5435 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5437 "Access Prohibited: off-limits",
5438 self.GET, self.url))
5440 # We should still be able to list the parent directory, in HTML...
5441 d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5442 def _check_dir_html2(body):
5443 self.failUnlessIn("<html>", body)
5444 self.failUnlessIn("blacklisted.txt</strike>", body)
5445 d.addCallback(_check_dir_html2)
5447 # ... and in JSON (used by CLI).
5448 d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5449 def _check_dir_json(res):
5450 data = simplejson.loads(res)
5451 self.failUnless(isinstance(data, list), data)
5452 self.failUnlessEqual(data[0], "dirnode")
5453 self.failUnless(isinstance(data[1], dict), data)
5454 self.failUnlessIn("children", data[1])
5455 self.failUnlessIn("blacklisted.txt", data[1]["children"])
5456 childdata = data[1]["children"]["blacklisted.txt"]
5457 self.failUnless(isinstance(childdata, list), data)
5458 self.failUnlessEqual(childdata[0], "filenode")
5459 self.failUnless(isinstance(childdata[1], dict), data)
5460 d.addCallback(_check_dir_json)
5462 def _unblacklist(ign):
5463 open(fn, "w").close()
5464 # the Blacklist object watches mtime to tell when the file has
5465 # changed, but on windows this test will run faster than the
5466 # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5467 # to force a reload.
5468 self.g.clients[0].blacklist.last_mtime -= 2.0
5469 d.addCallback(_unblacklist)
5471 # now a read should work
5472 d.addCallback(lambda ign: self.GET(self.url))
5473 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5475 # read again to exercise the blacklist-is-unchanged logic
5476 d.addCallback(lambda ign: self.GET(self.url))
5477 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5479 # now add a blacklisted directory, and make sure files under it are
5482 childnode = c0.create_node_from_uri(self.uri, None)
5483 return c0.create_dirnode({u"child": (childnode,{}) })
5484 d.addCallback(_add_dir)
5485 def _get_dircap(dn):
5486 self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5487 self.dir_url_base = "uri/"+dn.get_write_uri()
5488 self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5489 self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5490 self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5491 self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5492 d.addCallback(_get_dircap)
5493 d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5494 d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5495 d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5496 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5497 d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5498 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5499 d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5500 d.addCallback(lambda res: simplejson.loads(res)) # just check it decodes
5501 d.addCallback(lambda ign: self.GET(self.child_url))
5502 d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5504 def _block_dir(ign):
5506 f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5508 self.g.clients[0].blacklist.last_mtime -= 2.0
5509 d.addCallback(_block_dir)
5510 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5512 "Access Prohibited: dir-off-limits",
5513 self.GET, self.dir_url_base))
5514 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5516 "Access Prohibited: dir-off-limits",
5517 self.GET, self.dir_url_json1))
5518 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5520 "Access Prohibited: dir-off-limits",
5521 self.GET, self.dir_url_json2))
5522 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5524 "Access Prohibited: dir-off-limits",
5525 self.GET, self.dir_url_json_ro))
5526 d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5528 "Access Prohibited: dir-off-limits",
5529 self.GET, self.child_url))
5533 class CompletelyUnhandledError(Exception):
5535 class ErrorBoom(rend.Page):
5536 def beforeRender(self, ctx):
5537 raise CompletelyUnhandledError("whoops")